| 1 | package console |
| 2 | |
| 3 | // TestClaimConsoleMutationInvariant is the runtime half of the claim |
| 4 | // mutation invariant (plan 5 rev 4): |
| 5 | // |
| 6 | // With HostClaim on, no console code path mutates app-scoped state (bucket |
| 7 | // keys under the app's prefixes, its SQL rows, its backend and SES |
| 8 | // resources) without a live admission registration for that incarnation, |
| 9 | // or the exclusive operation that owns it. |
| 10 | // |
| 11 | // TestClaimConsoleRouteCensus (claim_routes_test.go) pins WHICH entries |
| 12 | // exist, from source. This test pins what each one DOES, at runtime: |
| 13 | // |
| 14 | // 1. Every registered entry, route or worker, from s.claimEntries.snapshot() |
| 15 | // on an enabled console, has exactly one fixture, and every fixture names |
| 16 | // a registered entry. A new door without a fixture fails here. |
| 17 | // 2. A fixture is exactly one of: read (reason), nonApp (reason), or door. |
| 18 | // There is no default. |
| 19 | // 3. A door runs twice per affected incarnation. With that incarnation |
| 20 | // closed under a foreign exclusive operation, the same authenticated, |
| 21 | // valid request leaves no raw effect on any affected app: no bucket key |
| 22 | // the production classifier scopes to it, no SQL row change, no outbound HTTP, SES, DNS, backend |
| 23 | // or versioned call at all, no write to its admission. Then, with every |
| 24 | // incarnation open, the same request must succeed, write its admission |
| 25 | // (the gate saw it) and reach at least one raw app effect. A 401, 404 or |
| 26 | // a refused body reaches no effect, so it can never count as coverage. |
| 27 | // 4. A read or nonApp entry has its own probe (claim_invariant_probes_test.go, |
| 28 | // the tables in claim_invariant_probes_*.go): a valid authenticated request |
| 29 | // against a staged resource, accepted only on the exact success status |
| 30 | // with an existing resource in the answer, no auth redirect, no error flash |
| 31 | // and a request that matched this very entry; a mutator also proves its |
| 32 | // effect from the world (a registry row, a bucket object, a SQL row, an |
| 33 | // account, a token). It must leave no app-scoped raw effect. A request the |
| 34 | // handler rejected (any status of 400 or more, an auth redirect, a 303 back |
| 35 | // with an error flash) is never coverage, and the detectors say so. |
| 36 | // An operation a deployment intentionally does not support is a refused |
| 37 | // fixture instead: the exact status and message, from claimUnsupported. |
| 38 | // 5. Every app-scoped writer the boundaries expose (bucket and versioned |
| 39 | // mutators by reflection over the installed boundary, every mutating |
| 40 | // backend method, every gated SQL table) is reached by at least one |
| 41 | // door's open run, or carries a written reason in claimWriterReasons. |
| 42 | // |
| 43 | // Effects are observed BELOW the boundaries (claim_invariant_world_test.go), |
| 44 | // so only what production let through is counted. |
| 45 | |
| 46 | import ( |
| 47 | "context" |
| 48 | "encoding/json" |
| 49 | "fmt" |
| 50 | "html" |
| 51 | "net/http" |
| 52 | "reflect" |
| 53 | "slices" |
| 54 | "sort" |
| 55 | "strings" |
| 56 | "sync" |
| 57 | "sync/atomic" |
| 58 | "testing" |
| 59 | "time" |
| 60 | |
| 61 | "github.com/carlosframework/platform/internal/dbreads" |
| 62 | "github.com/carlosframework/platform/internal/dnsrec" |
| 63 | "github.com/carlosframework/platform/internal/hostops" |
| 64 | "github.com/carlosframework/platform/internal/mail" |
| 65 | "github.com/carlosframework/platform/internal/objstore" |
| 66 | "github.com/carlosframework/platform/internal/release" |
| 67 | ) |
| 68 | |
| 69 | // claimFixture classifies one registered entry. Exactly one field is set. |
| 70 | type claimFixture struct { |
| 71 | read string // why this entry cannot mutate any state at all |
| 72 | nonApp string // why this entry mutates only deployment, account or person state |
| 73 | door *claimDoor // an app-scoped mutator, driven for real |
| 74 | refused *claimRefusalFixture // an intentionally unsupported app mutator, driven for real |
| 75 | } |
| 76 | |
| 77 | // claimRefusalFixture is an app-scoped door whose operation is intentionally |
| 78 | // unsupported on an enabled console (claimUnsupported). It is driven like a |
| 79 | // door, foreign and open, with the owner's real authenticated, valid |
| 80 | // request, and both runs must answer exactly status with exactly msg and |
| 81 | // leave no app or control effect at all. An auth refusal, a 404 or a |
| 82 | // generic error is never one of these: the census only admits 409 with a |
| 83 | // message from claimUnsupported. |
| 84 | type claimRefusalFixture struct { |
| 85 | apps []claimAppRef |
| 86 | setup func(t *testing.T, cx *claimCase) |
| 87 | // do sends the request and returns the final status and body. |
| 88 | do func(t *testing.T, cx *claimCase) (int, string) |
| 89 | status int |
| 90 | msg string |
| 91 | } |
| 92 | |
| 93 | // claimOfflineMoveUnsupportedMsg is MV-O20, what every offline move door |
| 94 | // answers on a console with host claims or Phase 1 moves on (offlineGate). |
| 95 | const claimOfflineMoveUnsupportedMsg = "offline moves are not available while host claims or Phase 1 moves are on" |
| 96 | |
| 97 | // claimUnsupported is the closed list of operations an enabled console |
| 98 | // refuses on purpose, keyed by the exact message production answers. Only |
| 99 | // these may be a refused fixture; every other app mutator is a door that |
| 100 | // must hold the gate and reach its effect. |
| 101 | var claimUnsupported = map[string]string{ |
| 102 | claimRunUnsupportedMsg: "carlos run launches a cmd unit that the claim's isolated root protocol has no reservation for (#250); " + |
| 103 | "every run route refuses before it reads or writes anything", |
| 104 | claimHostResolveUnsupportedMsg: "host-loss declaration/readmission changes claim roots; the existing service has no scoped lifecycle grant in this slice", |
| 105 | claimMoveUnsupportedMsg: "instance move request, cut and abort rewrite the record's move journal, " + |
| 106 | "which only the unbuilt host-claim Moves protocol may do under its own operation", |
| 107 | claimOfflineMoveUnsupportedMsg: "the offline operator move (MV-O20) is only for consoles with host claims and Phase 1 moves off; " + |
| 108 | "every offline door, the status view included, refuses after its operator gate and before it reads evidence or writes the record", |
| 109 | claimRefusalText(errClaimAliasDetachUnsupported): "removing an alias-only marker needs its own conditional effect operation; " + |
| 110 | "a first-route detach retires its record and is a door", |
| 111 | claimRefusalText(errClaimAliasCapabilitiesUnsupported): "wildcard and catch-all claims are written only by the legacy capability path", |
| 112 | claimRenameUnsupportedMsg: "rename rewrites the app's identity (SQL rows, prefixes, admission key); " + |
| 113 | "no operation gates the old and new names together, so every rename is refused", |
| 114 | claimTransferUnsupportedMsg: "account transfer rewrites the app's identity under another account; " + |
| 115 | "no operation gates both, so every transfer is refused", |
| 116 | claimMoveResolveUnavailableMsg: "the operator's door out of a stuck move (CARLOS_MOVES_PHASE1) has only its gate until " + |
| 117 | "the Moves resolution body lands; it answers this sentence and writes nothing", |
| 118 | claimDBReadResidencyMsg: "a forwarded db read needs the root signer, which a host-claim console never uses for it " + |
| 119 | "(db region forward spec §6): the reads route reads the record and answers main's residency sentence, writing nothing", |
| 120 | seedClaimConsoleRefusal().Msg: "seeded instances need nested boxes, and the claim-armed path waits for #250 (seeded-instances spec §3.4); " + |
| 121 | "every import door refuses before it reads the record, and the legacy route assignment before it reads anything", |
| 122 | } |
| 123 | |
| 124 | // claimDBReadResidencyMsg is main's residency sentence for the claim world's |
| 125 | // instance (in ie) on a console in the home region: what the forwarded db |
| 126 | // read answers on a host-claim console. claimDBReadBody is a valid read. |
| 127 | var ( |
| 128 | claimDBReadResidencyMsg = claimDBHost + " keeps its data in region ie. This console can only read data kept in the home region." |
| 129 | claimDBReadBody = func() string { |
| 130 | key, err := dbreads.NewReplyKey() |
| 131 | if err != nil { |
| 132 | panic(err) |
| 133 | } |
| 134 | return `{"op":"query","sql":"SELECT 1","reply_key":"` + dbreads.EncodeReplyKey(key.PublicKey()) + `"}` |
| 135 | }() |
| 136 | ) |
| 137 | |
| 138 | // claimLifecycleRoot is the classifier's NonApp reason for a lifecycle CAS |
| 139 | // root (hostops.ClassifyMutation): a host claim moved, which a refusal must |
| 140 | // never do. |
| 141 | const claimLifecycleRoot = "lifecycle CAS root" |
| 142 | |
| 143 | // claimDoor is one app-scoped mutator and the request that drives it. |
| 144 | type claimDoor struct { |
| 145 | // apps are the incarnations the request affects. apps[0] is the one the |
| 146 | // request names; it must write its admission in the open run. |
| 147 | apps []claimAppRef |
| 148 | // setup stages the world with every incarnation open, outside every |
| 149 | // measured window. It may drive other doors. |
| 150 | setup func(t *testing.T, cx *claimCase) |
| 151 | // do sends the authenticated, valid request (and any passkey step-up it |
| 152 | // asks for) and returns the final status. |
| 153 | do func(t *testing.T, cx *claimCase) int |
| 154 | // check, when set, proves from the world (not the status) that the |
| 155 | // handler's mutation path was reached: it runs after every foreign run |
| 156 | // (open=false, the mutation must NOT have happened) and after the open |
| 157 | // run (open=true, it must have). A non-empty answer is a problem. A |
| 158 | // rejected probe (bad session, CSRF or id) leaves the world unchanged, so |
| 159 | // it fails the open check however it answered. |
| 160 | check func(t *testing.T, cx *claimCase, open bool) string |
| 161 | } |
| 162 | |
| 163 | // claimWindow is what one measured request left behind. |
| 164 | type claimWindow struct { |
| 165 | status int |
| 166 | events []claimRawEvent |
| 167 | sqlChanged map[claimAppRef][]string |
| 168 | } |
| 169 | |
| 170 | // claimWriterReasons names app-scoped writers that no door's open run |
| 171 | // reaches in this fixture, each with the reason. Empty is the claim that |
| 172 | // every writer is reached. A new boundary method or gated table fails the |
| 173 | // coverage step until a door reaches it or a reason is written here. |
| 174 | var claimWriterReasons = map[string]string{ |
| 175 | "bucket/PutFrom": "the only console streaming caller is geoippublish.go, which writes deployment GeoIP blobs, not app state; app-prefixed fallback writes are checked by Put and the streaming new-door witness", |
| 176 | |
| 177 | "bucket/Delete": "the data boundary never sends an unconditional delete: Delete reads the exact revision, " + |
| 178 | "authorizes it and issues DeleteIfMatch, which bucket/DeleteIfMatch counts", |
| 179 | "versioned/PutIfAbsent": "no console door calls it: the console imports neither internal/park nor internal/hostbox, " + |
| 180 | "the only code that puts versioned objects, and its own versioned callers are the db door's reads and the purge sweep's DeleteVersion", |
| 181 | "versioned/CopyVersion": "no console door calls it: only internal/park (archive copies) and internal/hostbox call CopyVersion, " + |
| 182 | "and the console imports neither", |
| 183 | "sql/owners": "legacy per-app grants, written only by cmd/console's operator grant and revoke verbs " + |
| 184 | "(Store.GrantAt and Revoke); enabled operator DBs are guarded too and refuse these unbound legacy grants; no route or worker writes them", |
| 185 | "sql/mail_counts": "retired by the senders migration: nothing writes it, the one-shot backfill only reads it, " + |
| 186 | "and DropLegacyMailCounts drops it", |
| 187 | } |
| 188 | |
| 189 | // claimCoordinators are boundary methods that hold the gate for a caller and |
| 190 | // then run its function: they perform no raw write themselves, so they are |
| 191 | // not raw sinks. Every write inside the function reaches a real sink method, |
| 192 | // which the inventory counts. The reason is mandatory. |
| 193 | var claimCoordinators = map[string]string{ |
| 194 | "bucket/RunSQLIngest": "coordinator for the SQL adapter: admits one shared ingest and issues its receipt-bound effect; raw SQL is independently covered by the existing SQL door fixtures", |
| 195 | "bucket/RunAppMutation": "coordinator: registers or begins the exclusive app mutation, then runs the caller's function; " + |
| 196 | "it writes nothing raw itself (its admission and effect writes go through the typed control writer, " + |
| 197 | "and the function's own writes reach Put, Delete or DeleteIfMatch)", |
| 198 | } |
| 199 | |
| 200 | // claimCaseParallelism bounds how many door, refusal and probe subtests run |
| 201 | // at once. Each one builds its own console and measures what a request left |
| 202 | // behind by waiting for quiet windows (quiesce). Too many at once starve the |
| 203 | // CPU, and a handler goroutine that misses its window is measured with the |
| 204 | // wrong request; the bound keeps that contention down. |
| 205 | const claimCaseParallelism = 8 |
| 206 | |
| 207 | // claimParallelCases counts the claim cases running under t.Parallel. A |
| 208 | // fixture that swaps process-wide state (a package variable, the log |
| 209 | // output, a shared client's transport) calls claimMustRunAlone, which |
| 210 | // fails the case if any parallel case is running, itself included. A swap |
| 211 | // added to a fixture that is not listed serial therefore fails loudly |
| 212 | // instead of leaking into a neighbour's measurement. |
| 213 | var claimParallelCases atomic.Int32 |
| 214 | |
| 215 | func claimMustRunAlone(t *testing.T, what string) { |
| 216 | t.Helper() |
| 217 | if n := claimParallelCases.Load(); n > 0 { |
| 218 | t.Fatalf("%s swaps process-wide state but runs beside %d parallel claim case(s): list it serial (claimSerialDoors, or the probe's serial field)", what, n) |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | // claimParallelSlot takes a slot for a parallel claim case and counts it |
| 223 | // as running until its own cleanup is done. |
| 224 | func claimParallelSlot(t *testing.T, slots chan struct{}) { |
| 225 | slots <- struct{}{} |
| 226 | // Registered first, so it runs last: the slot is held until the case's |
| 227 | // own cleanup (servers, store) is done. |
| 228 | t.Cleanup(func() { <-slots }) |
| 229 | claimParallelCases.Add(1) |
| 230 | t.Cleanup(func() { claimParallelCases.Add(-1) }) |
| 231 | } |
| 232 | |
| 233 | // claimSerialDoors are the door and refusal fixtures that must not run in |
| 234 | // parallel, each with the reason. They run first, one at a time, before any |
| 235 | // parallel door starts. A fixture whose staging touches process-wide state |
| 236 | // (the environment, the log output, a package variable) belongs here. |
| 237 | var claimSerialDoors = map[string]string{ |
| 238 | "route POST /api/cli/relay/rotate": "relayEnabled and relayStageMaster stage the region's relay masters " + |
| 239 | "through t.Setenv: process environment, which t.Parallel forbids", |
| 240 | "route POST /api/cli/apps/{account}/{app}/relay": "relayArmed stages the region's relay master through t.Setenv: " + |
| 241 | "process environment, which t.Parallel forbids", |
| 242 | "route POST /api/cli/apps/{account}/{app}/relay/adopted": "relayEnabled stages the region's relay master through t.Setenv: " + |
| 243 | "process environment, which t.Parallel forbids", |
| 244 | "route POST /api/cli/apps/{account}/{app}/relay/disable": "relayEnabled stages the region's relay master through t.Setenv: " + |
| 245 | "process environment, which t.Parallel forbids", |
| 246 | "worker StartRelay": "relayEnabled stages the region's relay master through t.Setenv: " + |
| 247 | "process environment, which t.Parallel forbids", |
| 248 | "worker StartMailEventPush": "swaps mailPushClient.Transport, a package variable, for its recorder", |
| 249 | } |
| 250 | |
| 251 | // claimSerialDoorCensus: every serial entry names a door or refusal fixture |
| 252 | // and carries its reason. |
| 253 | func claimSerialDoorCensus(fixtures map[string]claimFixture, serial map[string]string) []string { |
| 254 | var problems []string |
| 255 | for key, reason := range serial { |
| 256 | f, ok := fixtures[key] |
| 257 | switch { |
| 258 | case !ok || (f.door == nil && f.refused == nil): |
| 259 | problems = append(problems, fmt.Sprintf("claimSerialDoors names %s, which is not a door or refusal fixture", key)) |
| 260 | case strings.TrimSpace(reason) == "": |
| 261 | problems = append(problems, fmt.Sprintf("claimSerialDoors names %s with no reason", key)) |
| 262 | } |
| 263 | } |
| 264 | sort.Strings(problems) |
| 265 | return problems |
| 266 | } |
| 267 | |
| 268 | func TestClaimConsoleMutationInvariant(t *testing.T) { |
| 269 | census := newClaimCase(t) |
| 270 | if !census.srv.cfg.HostClaim { |
| 271 | t.Fatal("the invariant console is not enabled") |
| 272 | } |
| 273 | if problems := claimFixtureCensus(census.srv.claimEntries.snapshot(), claimFixtures); len(problems) != 0 { |
| 274 | for _, p := range problems { |
| 275 | t.Error(p) |
| 276 | } |
| 277 | t.FailNow() |
| 278 | } |
| 279 | if problems := claimProbeCensus(claimFixtures, claimProbes); len(problems) != 0 { |
| 280 | for _, p := range problems { |
| 281 | t.Error(p) |
| 282 | } |
| 283 | t.FailNow() |
| 284 | } |
| 285 | if problems := claimVariantCensus(census.srv.claimEntries.snapshot(), claimFixtures, claimLifecycleVariants); len(problems) != 0 { |
| 286 | for _, p := range problems { |
| 287 | t.Error(p) |
| 288 | } |
| 289 | t.FailNow() |
| 290 | } |
| 291 | inventory := claimWriterInventory(census.srv) |
| 292 | |
| 293 | keys := make([]string, 0, len(claimFixtures)) |
| 294 | for k := range claimFixtures { |
| 295 | keys = append(keys, k) |
| 296 | } |
| 297 | sort.Strings(keys) |
| 298 | |
| 299 | if problems := claimSerialDoorCensus(claimFixtures, claimSerialDoors); len(problems) != 0 { |
| 300 | for _, p := range problems { |
| 301 | t.Error(p) |
| 302 | } |
| 303 | t.FailNow() |
| 304 | } |
| 305 | |
| 306 | // Doors and refusals, each on its own console. The group returns only |
| 307 | // when every one of them has finished, parallel ones included, so the |
| 308 | // writer check below sees every door's open run. |
| 309 | var ( |
| 310 | reachedMu sync.Mutex |
| 311 | reached = map[string]bool{} |
| 312 | ) |
| 313 | t.Run("doors", func(t *testing.T) { |
| 314 | slots := make(chan struct{}, claimCaseParallelism) |
| 315 | run := func(t *testing.T, f claimFixture) { |
| 316 | if f.refused != nil { |
| 317 | for _, p := range runClaimRefusal(t, f.refused) { |
| 318 | t.Error(p) |
| 319 | } |
| 320 | return |
| 321 | } |
| 322 | problems, writers := runClaimDoor(t, f.door) |
| 323 | for _, p := range problems { |
| 324 | t.Error(p) |
| 325 | } |
| 326 | reachedMu.Lock() |
| 327 | for _, w := range writers { |
| 328 | reached[w] = true |
| 329 | } |
| 330 | reachedMu.Unlock() |
| 331 | } |
| 332 | // The serial ones first, one at a time, while nothing else runs: |
| 333 | // a parallel subtest only starts once this function has returned. |
| 334 | for _, key := range keys { |
| 335 | if f := claimFixtures[key]; (f.door != nil || f.refused != nil) && claimSerialDoors[key] != "" { |
| 336 | t.Run(key, func(t *testing.T) { run(t, f) }) |
| 337 | } |
| 338 | } |
| 339 | for _, key := range keys { |
| 340 | if f := claimFixtures[key]; (f.door != nil || f.refused != nil) && claimSerialDoors[key] == "" { |
| 341 | t.Run(key, func(t *testing.T) { |
| 342 | t.Parallel() |
| 343 | claimParallelSlot(t, slots) |
| 344 | run(t, f) |
| 345 | }) |
| 346 | } |
| 347 | } |
| 348 | }) |
| 349 | |
| 350 | // Lifecycle variants: further actions on an entry above whose only |
| 351 | // effect is a host claim written under their own registration grant |
| 352 | // (claim_invariant_wakereset_test.go). |
| 353 | runClaimLifecycleVariants(t) |
| 354 | |
| 355 | // Reads and nonApp entries are probed with their own valid request on |
| 356 | // their own console (claim_invariant_probes_test.go): a request the |
| 357 | // handler turned away is not coverage, and a mutator proves its effect |
| 358 | // from the world. They run in parallel too, after every door, except the |
| 359 | // probes marked serial (claimProbe.serial), which run alone first. |
| 360 | t.Run("probes", func(t *testing.T) { |
| 361 | runClaimProbes(t, claimFixtures, claimProbes, claimCaseParallelism) |
| 362 | }) |
| 363 | |
| 364 | var unreached []string |
| 365 | for w := range inventory { |
| 366 | if !reached[w] && claimWriterReasons[w] == "" { |
| 367 | unreached = append(unreached, w) |
| 368 | } |
| 369 | } |
| 370 | for w := range claimWriterReasons { |
| 371 | if !inventory[w] { |
| 372 | t.Errorf("claimWriterReasons names %s, which no boundary exposes any more", w) |
| 373 | } |
| 374 | } |
| 375 | sort.Strings(unreached) |
| 376 | for _, w := range unreached { |
| 377 | t.Errorf("app-scoped writer %s is reached by no door's open run and has no reason in claimWriterReasons", w) |
| 378 | } |
| 379 | } |
| 380 | |
| 381 | // claimFixtureCensus compares the registered entries with the fixture table, |
| 382 | // exactly, and checks each fixture is one kind with its reason or door. |
| 383 | func claimFixtureCensus(entries []claimEntry, fixtures map[string]claimFixture) []string { |
| 384 | var problems []string |
| 385 | got := map[string]bool{} |
| 386 | for _, e := range entries { |
| 387 | key := e.Kind + " " + e.Pattern |
| 388 | got[key] = true |
| 389 | if _, ok := fixtures[key]; !ok { |
| 390 | problems = append(problems, fmt.Sprintf("registered %s (handler %s) has no claim fixture: add a door, or a read or nonApp reason", key, e.Handler)) |
| 391 | } |
| 392 | } |
| 393 | for key, f := range fixtures { |
| 394 | if !got[key] { |
| 395 | problems = append(problems, fmt.Sprintf("claim fixture %s names no registered entry", key)) |
| 396 | } |
| 397 | kinds := 0 |
| 398 | if strings.TrimSpace(f.read) != "" { |
| 399 | kinds++ |
| 400 | } |
| 401 | if strings.TrimSpace(f.nonApp) != "" { |
| 402 | kinds++ |
| 403 | } |
| 404 | if f.door != nil { |
| 405 | kinds++ |
| 406 | if len(f.door.apps) == 0 || f.door.do == nil { |
| 407 | problems = append(problems, fmt.Sprintf("claim fixture %s is a door with no affected app or no request", key)) |
| 408 | } |
| 409 | } |
| 410 | if r := f.refused; r != nil { |
| 411 | kinds++ |
| 412 | switch { |
| 413 | case len(r.apps) == 0 || r.do == nil: |
| 414 | problems = append(problems, fmt.Sprintf("claim fixture %s is a refusal with no affected app or no request", key)) |
| 415 | case r.status != http.StatusConflict: |
| 416 | problems = append(problems, fmt.Sprintf("claim fixture %s is a refusal answering %d: only the explicit 409 of an intentionally "+ |
| 417 | "unsupported operation can be one (never 401, 403, 404 or a generic error)", key, r.status)) |
| 418 | case strings.TrimSpace(claimUnsupported[r.msg]) == "": |
| 419 | problems = append(problems, fmt.Sprintf("claim fixture %s is a refusal with message %q, which is not an intentionally "+ |
| 420 | "unsupported operation in claimUnsupported: make it a door", key, r.msg)) |
| 421 | } |
| 422 | } |
| 423 | if kinds != 1 { |
| 424 | problems = append(problems, fmt.Sprintf("claim fixture %s must be exactly one of read, nonApp, door or refused (has %d)", key, kinds)) |
| 425 | } |
| 426 | } |
| 427 | sort.Strings(problems) |
| 428 | return problems |
| 429 | } |
| 430 | |
| 431 | // runClaimDoor drives one door: setup, one foreign-exclusive run per |
| 432 | // affected incarnation, then the open run. It returns the problems and the |
| 433 | // writers the open run reached. |
| 434 | func runClaimDoor(t *testing.T, d *claimDoor, opts ...func(*Config)) ([]string, []string) { |
| 435 | t.Helper() |
| 436 | cx := newClaimCaseWith(t, opts...) |
| 437 | if d.setup != nil { |
| 438 | d.setup(t, cx) |
| 439 | } |
| 440 | cx.quiesce() |
| 441 | var problems []string |
| 442 | for _, ref := range d.apps { |
| 443 | restore := cx.foreign(t, ref) |
| 444 | w := cx.measure(t, d) |
| 445 | problems = append(problems, claimForeignProblems(ref, d.apps, w)...) |
| 446 | if d.check != nil { |
| 447 | if p := d.check(t, cx, false); p != "" { |
| 448 | problems = append(problems, "foreign exclusive on "+ref.String()+": "+p) |
| 449 | } |
| 450 | } |
| 451 | restore() |
| 452 | } |
| 453 | w := cx.measure(t, d) |
| 454 | problems = append(problems, claimOpenProblems(d.apps, w)...) |
| 455 | if d.check != nil { |
| 456 | if p := d.check(t, cx, true); p != "" { |
| 457 | problems = append(problems, "open run: "+p) |
| 458 | } |
| 459 | } |
| 460 | return problems, claimWritersOf(d.apps, w) |
| 461 | } |
| 462 | |
| 463 | // runClaimRefusal drives one intentionally unsupported door like a door: |
| 464 | // setup, one foreign-exclusive run per affected incarnation, then the open |
| 465 | // run. Every run must answer the exact refusal and leave nothing behind. |
| 466 | func runClaimRefusal(t *testing.T, f *claimRefusalFixture, opts ...func(*Config)) []string { |
| 467 | t.Helper() |
| 468 | cx := newClaimCaseWith(t, opts...) |
| 469 | if f.setup != nil { |
| 470 | f.setup(t, cx) |
| 471 | } |
| 472 | cx.quiesce() |
| 473 | var problems []string |
| 474 | run := func(label string) { |
| 475 | var body string |
| 476 | w := cx.measureRun(t, f.apps, func() int { |
| 477 | status, out := f.do(t, cx) |
| 478 | body = out |
| 479 | return status |
| 480 | }) |
| 481 | problems = append(problems, claimRefusalProblems(label, f, w, body)...) |
| 482 | } |
| 483 | for _, ref := range f.apps { |
| 484 | restore := cx.foreign(t, ref) |
| 485 | run("foreign exclusive on " + ref.String()) |
| 486 | restore() |
| 487 | } |
| 488 | run("open run") |
| 489 | return problems |
| 490 | } |
| 491 | |
| 492 | // claimRefusalProblems: the run answered exactly the unsupported refusal |
| 493 | // and reached no app effect, no control object and no host claim. |
| 494 | func claimRefusalProblems(label string, f *claimRefusalFixture, w claimWindow, body string) []string { |
| 495 | var problems []string |
| 496 | say := func(format string, args ...any) { |
| 497 | problems = append(problems, label+": "+fmt.Sprintf(format, args...)) |
| 498 | } |
| 499 | if w.status != f.status { |
| 500 | say("the authenticated valid request answered %d, want the explicit %d refusal", w.status, f.status) |
| 501 | } |
| 502 | if !claimRefusalSays(body, f.msg) { |
| 503 | if len(body) > 300 { |
| 504 | body = body[:300] + "..." |
| 505 | } |
| 506 | say("the answer %q is not exactly the unsupported refusal %q", body, f.msg) |
| 507 | } |
| 508 | for _, e := range w.events { |
| 509 | switch { |
| 510 | case e.Control: |
| 511 | say("the refusal wrote control object %s", e.Key) |
| 512 | case e.Err != nil: |
| 513 | say("raw %s reached a key the production classifier refuses (%v)", e, e.Err) |
| 514 | case e.Unscoped: |
| 515 | say("raw %s reached an outbound sink or a data bucket", e) |
| 516 | case len(e.Scopes) != 0: |
| 517 | say("raw %s is app-scoped (%v)", e, e.Scopes) |
| 518 | case e.NonApp == claimLifecycleRoot: |
| 519 | say("raw %s moved a lifecycle root", e) |
| 520 | } |
| 521 | } |
| 522 | for ref, changed := range w.sqlChanged { |
| 523 | say("SQL rows of %s changed in %v", ref, changed) |
| 524 | } |
| 525 | return problems |
| 526 | } |
| 527 | |
| 528 | // claimRefusalSays: a JSON error answer carries exactly msg; a rendered |
| 529 | // page carries msg as its escaped text. |
| 530 | func claimRefusalSays(body, msg string) bool { |
| 531 | var reply struct { |
| 532 | Error *string `json:"error"` |
| 533 | } |
| 534 | if json.Unmarshal([]byte(body), &reply) == nil { |
| 535 | return reply.Error != nil && *reply.Error == msg |
| 536 | } |
| 537 | return strings.Contains(body, html.EscapeString(msg)) |
| 538 | } |
| 539 | |
| 540 | // measure sends the door's request once and returns what it left behind. |
| 541 | func (cx *claimCase) measure(t *testing.T, d *claimDoor) claimWindow { |
| 542 | t.Helper() |
| 543 | return cx.measureRun(t, d.apps, func() int { return d.do(t, cx) }) |
| 544 | } |
| 545 | |
| 546 | // measureRun sends one request through do and returns what it left behind |
| 547 | // on apps. The CLI rate limiters are fresh for every run: a 429 is not a |
| 548 | // gate. |
| 549 | func (cx *claimCase) measureRun(t *testing.T, apps []claimAppRef, do func() int) claimWindow { |
| 550 | t.Helper() |
| 551 | cx.srv.cli = newCLILimits() |
| 552 | before := map[claimAppRef]map[string]string{} |
| 553 | for _, ref := range apps { |
| 554 | before[ref] = cx.sqlDigest(t, ref) |
| 555 | } |
| 556 | m := cx.rec.mark() |
| 557 | status := do() |
| 558 | cx.quiesce() |
| 559 | w := claimWindow{status: status, events: cx.rec.since(m), sqlChanged: map[claimAppRef][]string{}} |
| 560 | for _, ref := range apps { |
| 561 | if changed := claimChangedTables(before[ref], cx.sqlDigest(t, ref)); len(changed) != 0 { |
| 562 | w.sqlChanged[ref] = changed |
| 563 | } |
| 564 | } |
| 565 | return w |
| 566 | } |
| 567 | |
| 568 | // claimForeignProblems: with foreign holding an exclusive operation on one |
| 569 | // affected incarnation, nothing may reach any affected app. |
| 570 | func claimForeignProblems(foreign claimAppRef, refs []claimAppRef, w claimWindow) []string { |
| 571 | var problems []string |
| 572 | say := func(format string, args ...any) { |
| 573 | problems = append(problems, fmt.Sprintf("foreign exclusive on %s: ", foreign)+fmt.Sprintf(format, args...)) |
| 574 | } |
| 575 | for _, e := range w.events { |
| 576 | switch { |
| 577 | case e.Control && e.touches([]claimAppRef{foreign}): |
| 578 | say("the request wrote %s's canonical control object %s", foreign, e.Key) |
| 579 | case e.Control: |
| 580 | // The shared mail gate, or another affected app's registration |
| 581 | // taken and released around the refused preflight: gate state, |
| 582 | // not an app effect. |
| 583 | case e.Err != nil: |
| 584 | say("raw %s reached a key the production classifier refuses (%v)", e, e.Err) |
| 585 | case e.Unscoped: |
| 586 | say("raw %s reached an outbound sink or a data bucket", e) |
| 587 | case e.NonApp == claimLifecycleRoot: |
| 588 | say("raw %s moved a lifecycle root while an affected app was foreign-owned", e) |
| 589 | case e.touches(refs): |
| 590 | say("raw %s reached an affected app", e) |
| 591 | } |
| 592 | } |
| 593 | for _, ref := range refs { |
| 594 | if changed := w.sqlChanged[ref]; len(changed) != 0 { |
| 595 | say("SQL rows of %s changed in %v", ref, changed) |
| 596 | } |
| 597 | } |
| 598 | return problems |
| 599 | } |
| 600 | |
| 601 | // claimOpenProblems: with every incarnation open, the request must be |
| 602 | // admitted by the gate and reach at least one raw app effect. |
| 603 | func claimOpenProblems(refs []claimAppRef, w claimWindow) []string { |
| 604 | var problems []string |
| 605 | primary := refs[0] |
| 606 | if w.status == 0 || w.status >= 400 { |
| 607 | problems = append(problems, fmt.Sprintf("open run: the authenticated valid request answered %d", w.status)) |
| 608 | } |
| 609 | admitted, effect := false, false |
| 610 | for _, e := range w.events { |
| 611 | switch { |
| 612 | case e.admissionWrite(primary): |
| 613 | admitted = true |
| 614 | case e.Control: |
| 615 | case e.Err != nil: |
| 616 | problems = append(problems, fmt.Sprintf("open run: raw %s reached a key the production classifier refuses (%v)", e, e.Err)) |
| 617 | case e.Unscoped: |
| 618 | effect = true |
| 619 | case e.touches(refs): |
| 620 | effect = true |
| 621 | } |
| 622 | } |
| 623 | for _, ref := range refs { |
| 624 | if len(w.sqlChanged[ref]) != 0 { |
| 625 | effect = true |
| 626 | } |
| 627 | } |
| 628 | if !admitted { |
| 629 | problems = append(problems, fmt.Sprintf("open run: no write to %s's admission, so the request never registered with the gate", primary)) |
| 630 | } |
| 631 | if !effect { |
| 632 | problems = append(problems, "open run: the request reached no raw app effect (bucket, SQL, versioned, HTTP, SES, DNS or backend)") |
| 633 | } |
| 634 | return problems |
| 635 | } |
| 636 | |
| 637 | // claimWritersOf names the writers an open run reached. |
| 638 | func claimWritersOf(refs []claimAppRef, w claimWindow) []string { |
| 639 | set := map[string]bool{} |
| 640 | for _, e := range w.events { |
| 641 | switch { |
| 642 | case e.Control || e.Err != nil: |
| 643 | case !e.Unscoped && !e.touches(refs): |
| 644 | case e.Surface == "bucket": |
| 645 | set["bucket/"+e.Method] = true |
| 646 | case strings.HasPrefix(e.Surface, "versioned:"): |
| 647 | set["versioned/"+e.Method] = true |
| 648 | default: |
| 649 | set[e.Surface+"/"+e.Method] = true |
| 650 | } |
| 651 | } |
| 652 | for _, changed := range w.sqlChanged { |
| 653 | for _, table := range changed { |
| 654 | set["sql/"+table] = true |
| 655 | } |
| 656 | } |
| 657 | out := make([]string, 0, len(set)) |
| 658 | for k := range set { |
| 659 | out = append(out, k) |
| 660 | } |
| 661 | sort.Strings(out) |
| 662 | return out |
| 663 | } |
| 664 | |
| 665 | // claimWriterInventory is every app-scoped writer the installed boundaries |
| 666 | // expose. The bucket and versioned method sets are read off the INSTALLED |
| 667 | // boundary by reflection, so a writer method added to it joins the list. |
| 668 | func claimWriterInventory(s *Server) map[string]bool { |
| 669 | inv := map[string]bool{"http/POST": true} |
| 670 | methods := func(prefix string, v any, reads ...string) { |
| 671 | rt := reflect.TypeOf(v) |
| 672 | for i := 0; i < rt.NumMethod(); i++ { |
| 673 | name := rt.Method(i).Name |
| 674 | if slices.Contains(reads, name) { |
| 675 | continue |
| 676 | } |
| 677 | if reason, ok := claimCoordinators[prefix+name]; ok && strings.TrimSpace(reason) != "" { |
| 678 | continue |
| 679 | } |
| 680 | inv[prefix+name] = true |
| 681 | } |
| 682 | } |
| 683 | methods("bucket/", s.cfg.Releases, "Get", "List", "ListDirs", "Open", "ListMod", "ListSizes", "ListPage", "BucketShapedListDirs", "GetIfChanged", "ListETags") |
| 684 | methods("versioned/", s.cfg.DBVersions, "ListVersions", "OpenVersion", "GetVersion", "OpenVersionRange") |
| 685 | for _, b := range []struct { |
| 686 | surface string |
| 687 | iface reflect.Type |
| 688 | reads []string |
| 689 | }{ |
| 690 | {"objstore.Provisioner", reflect.TypeFor[objstore.Provisioner](), nil}, |
| 691 | {"objstore.Scanner", reflect.TypeFor[objstore.Scanner](), []string{"Capacity", "Enforcement", "Provenance", "Status", "Verdict"}}, |
| 692 | {"mail.Provisioner", reflect.TypeFor[mail.Provisioner](), []string{"IdentityStatus"}}, |
| 693 | {"dnsrec.Writer", reflect.TypeFor[dnsrec.Writer](), nil}, |
| 694 | } { |
| 695 | for i := 0; i < b.iface.NumMethod(); i++ { |
| 696 | if name := b.iface.Method(i).Name; !slices.Contains(b.reads, name) { |
| 697 | inv[b.surface+"/"+name] = true |
| 698 | } |
| 699 | } |
| 700 | } |
| 701 | for name, tb := range claimSQLTables { |
| 702 | if tb.NonApp == "" { |
| 703 | inv["sql/"+name] = true |
| 704 | } |
| 705 | } |
| 706 | return inv |
| 707 | } |
| 708 | |
| 709 | // ---- reads and nonApp entries ---------------------------------------------- |
| 710 | |
| 711 | // The generic request this section used to send every entry ({} for a body, |
| 712 | // "claim-probe" for an id) is gone: runClaimProbe (claim_invariant_probes_test.go) |
| 713 | // sends each one its own valid request and judges the answer and the world. |
| 714 | |
| 715 | // ---- the enabled constructor ------------------------------------------------ |
| 716 | |
| 717 | // TestClaimConstructorInstallsEveryBoundary: the console built with the flag |
| 718 | // on from the start never hands a handler a raw writer. |
| 719 | func TestClaimConstructorInstallsEveryBoundary(t *testing.T) { |
| 720 | cx := newClaimCase(t) |
| 721 | s := cx.srv |
| 722 | if !s.cfg.HostClaim || !cx.st.claimGated() { |
| 723 | t.Fatalf("HostClaim %v, SQL gate armed %v: want both", s.cfg.HostClaim, cx.st.claimGated()) |
| 724 | } |
| 725 | for _, c := range []struct { |
| 726 | name string |
| 727 | installed any |
| 728 | raw any |
| 729 | }{ |
| 730 | {"Releases", s.cfg.Releases, cx.rawStore}, |
| 731 | {"DBVersions", s.cfg.DBVersions, cx.rawVersions}, |
| 732 | {"StoreProvisioner", s.cfg.StoreProvisioner, cx.rawProv}, |
| 733 | {"StoreScanner", s.cfg.StoreScanner, cx.rawScanner}, |
| 734 | {"MailProvisioner", s.cfg.MailProvisioner, cx.rawMail}, |
| 735 | {"DNSWriter", s.cfg.DNSWriter, cx.rawDNS}, |
| 736 | } { |
| 737 | if c.installed == nil || c.installed == c.raw { |
| 738 | t.Errorf("Config.%s is the raw writer (or nil) after ConfigureClaimRuntime: %T", c.name, c.installed) |
| 739 | } |
| 740 | rt := reflect.TypeOf(c.installed) |
| 741 | if rt != nil && rt.Kind() == reflect.Pointer { |
| 742 | rt = rt.Elem() |
| 743 | } |
| 744 | if rt != nil && rt.Kind() == reflect.Struct { |
| 745 | for i := 0; i < rt.NumField(); i++ { |
| 746 | if rt.Field(i).Anonymous { |
| 747 | t.Errorf("Config.%s (%s) embeds %s: a promoted method would bypass the gate", c.name, rt, rt.Field(i).Type) |
| 748 | } |
| 749 | } |
| 750 | } |
| 751 | } |
| 752 | ctx := context.Background() |
| 753 | m := cx.rec.mark() |
| 754 | if _, err := s.cfg.Releases.Put(ctx, "apps/bab/titogo/claim-ungated.json", []byte(`{}`), ""); err == nil { |
| 755 | t.Error("an ungated app-key Put through Config.Releases succeeded") |
| 756 | } |
| 757 | if _, err := s.cfg.Releases.Put(ctx, admissionKeyOf("bab", "titogo"), []byte(`{}`), "x"); err == nil { |
| 758 | t.Error("Config.Releases wrote a canonical admission object") |
| 759 | } |
| 760 | if err := s.cfg.Releases.Delete(ctx, admissionKeyOf("bab", "titogo")); err == nil { |
| 761 | t.Error("Config.Releases deleted a canonical admission object") |
| 762 | } |
| 763 | if _, err := s.cfg.StoreProvisioner.EnsureBucket(ctx, "carlos-bab-titogo"); err == nil { |
| 764 | t.Error("an ungated EnsureBucket through Config.StoreProvisioner succeeded") |
| 765 | } |
| 766 | if _, err := s.cfg.DBVersions.PutIfAbsent(ctx, claimFxBucket, "apps/bab/titogo/claim-ungated.json", []byte(`{}`)); err == nil { |
| 767 | t.Error("an ungated PutIfAbsent through Config.DBVersions succeeded") |
| 768 | } |
| 769 | if _, err := s.store.db.ExecContext(ctx, `UPDATE apps SET claim_revision = claim_revision + 1 WHERE app = 'titogo'`); err == nil { |
| 770 | t.Error("an ungated UPDATE of an app row succeeded on the console's store") |
| 771 | } |
| 772 | cx.quiesce() |
| 773 | if events := cx.rec.since(m); len(events) != 0 { |
| 774 | t.Errorf("ungated writes reached the raw writers: %v", events) |
| 775 | } |
| 776 | } |
| 777 | |
| 778 | // ---- new-door witnesses ----------------------------------------------------- |
| 779 | |
| 780 | // claimWitnessHandler answers 401 to anything but the case's owner token, |
| 781 | // so the witnesses below are authenticated doors like every real one. |
| 782 | func claimWitnessHandler(cx *claimCase, write func(r *http.Request) error) http.HandlerFunc { |
| 783 | return func(w http.ResponseWriter, r *http.Request) { |
| 784 | if _, ok := cx.st.CLIPrincipal(bearer(r), time.Now()); !ok { |
| 785 | http.Error(w, "not signed in", http.StatusUnauthorized) |
| 786 | return |
| 787 | } |
| 788 | if err := write(r); err != nil { |
| 789 | http.Error(w, err.Error(), http.StatusConflict) |
| 790 | return |
| 791 | } |
| 792 | w.WriteHeader(http.StatusOK) |
| 793 | } |
| 794 | } |
| 795 | |
| 796 | // claimWitnessRefusalHandler is an intentionally unsupported witness door: |
| 797 | // 401 to anything but the owner token, otherwise write (if any) and then |
| 798 | // production's exact move refusal. |
| 799 | func claimWitnessRefusalHandler(cx *claimCase, write func(r *http.Request) error) http.HandlerFunc { |
| 800 | return func(w http.ResponseWriter, r *http.Request) { |
| 801 | if _, ok := cx.st.CLIPrincipal(bearer(r), time.Now()); !ok { |
| 802 | http.Error(w, "not signed in", http.StatusUnauthorized) |
| 803 | return |
| 804 | } |
| 805 | if write != nil { |
| 806 | _ = write(r) |
| 807 | } |
| 808 | jsonError(w, http.StatusConflict, claimMoveUnsupportedMsg) |
| 809 | } |
| 810 | } |
| 811 | |
| 812 | var claimWitnessApps = []claimAppRef{{Account: "bab", App: "titogo"}} |
| 813 | |
| 814 | func claimWitnessDoor(path string, write func(cx *claimCase) func(r *http.Request) error, authed bool) *claimDoor { |
| 815 | return &claimDoor{ |
| 816 | apps: claimWitnessApps, |
| 817 | setup: func(t *testing.T, cx *claimCase) { |
| 818 | cx.srv.registerRoute("POST "+path, claimWitnessHandler(cx, write(cx))) |
| 819 | }, |
| 820 | do: func(t *testing.T, cx *claimCase) int { |
| 821 | token := "" |
| 822 | if authed { |
| 823 | token = cx.bearer |
| 824 | } |
| 825 | status, _, _ := rawReq(t, http.MethodPost, cx.ts.URL+path, token, "{}") |
| 826 | return status |
| 827 | }, |
| 828 | } |
| 829 | } |
| 830 | |
| 831 | // The structural witnesses: each one is a door added after this test was |
| 832 | // written. The first must fail the census, the ungated ones must fail their |
| 833 | // open run, the gated one must pass both runs, and the detector must see a |
| 834 | // leak when one is shown to it. |
| 835 | func TestClaimConsoleMutationInvariantWitnesses(t *testing.T) { |
| 836 | const witnessKey = "titogo-claim-witness.json" |
| 837 | appKey := "apps/bab/titogo/" + witnessKey |
| 838 | |
| 839 | t.Run("missing fixture", func(t *testing.T) { |
| 840 | cx := newClaimCase(t) |
| 841 | cx.srv.registerRoute("POST /claim-witness/missing", cx.srv.handleHome) |
| 842 | problems := claimFixtureCensus(cx.srv.claimEntries.snapshot(), claimFixtures) |
| 843 | if !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, "route POST /claim-witness/missing") }) { |
| 844 | t.Fatalf("a route with no fixture passed the census: %v", problems) |
| 845 | } |
| 846 | withFixture := map[string]claimFixture{} |
| 847 | for k, v := range claimFixtures { |
| 848 | withFixture[k] = v |
| 849 | } |
| 850 | withFixture["route POST /claim-witness/missing"] = claimFixture{} |
| 851 | if p := claimFixtureCensus(cx.srv.claimEntries.snapshot(), withFixture); !slices.ContainsFunc(p, func(p string) bool { |
| 852 | return strings.Contains(p, "must be exactly one of") |
| 853 | }) { |
| 854 | t.Fatalf("a fixture with no classification passed the census: %v", p) |
| 855 | } |
| 856 | }) |
| 857 | |
| 858 | ungated := map[string]func(cx *claimCase) func(r *http.Request) error{ |
| 859 | "streaming bucket": func(cx *claimCase) func(r *http.Request) error { |
| 860 | return func(r *http.Request) error { |
| 861 | _, err := release.PutObjectFrom(r.Context(), cx.srv.cfg.Releases, appKey, strings.NewReader("{}"), 2, "") |
| 862 | return err |
| 863 | } |
| 864 | }, |
| 865 | "bucket": func(cx *claimCase) func(r *http.Request) error { |
| 866 | return func(r *http.Request) error { |
| 867 | _, err := cx.srv.cfg.Releases.Put(r.Context(), appKey, []byte(`{"ungated":true}`), "") |
| 868 | return err |
| 869 | } |
| 870 | }, |
| 871 | "sql": func(cx *claimCase) func(r *http.Request) error { |
| 872 | return func(r *http.Request) error { |
| 873 | _, err := cx.srv.store.db.ExecContext(r.Context(), |
| 874 | `INSERT INTO audit (app, address, action, created_at, account_id) VALUES ('titogo', ?, 'claim-witness', 1, ?)`, |
| 875 | claimFxOwner, OpsAccountID) |
| 876 | return err |
| 877 | } |
| 878 | }, |
| 879 | "backend": func(cx *claimCase) func(r *http.Request) error { |
| 880 | return func(r *http.Request) error { |
| 881 | _, err := cx.srv.cfg.StoreProvisioner.EnsureBucket(r.Context(), "carlos-bab-titogo") |
| 882 | return err |
| 883 | } |
| 884 | }, |
| 885 | "versioned": func(cx *claimCase) func(r *http.Request) error { |
| 886 | return func(r *http.Request) error { |
| 887 | _, err := cx.srv.cfg.DBVersions.PutIfAbsent(r.Context(), claimFxBucket, appKey, []byte(`{}`)) |
| 888 | return err |
| 889 | } |
| 890 | }, |
| 891 | } |
| 892 | for name, write := range ungated { |
| 893 | t.Run("fixture but ungated "+name+" mutator", func(t *testing.T) { |
| 894 | problems, _ := runClaimDoor(t, claimWitnessDoor("/claim-witness/ungated-"+name, write, true)) |
| 895 | if !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, "reached no raw app effect") }) { |
| 896 | t.Fatalf("an ungated %s mutator passed the invariant: %v", name, problems) |
| 897 | } |
| 898 | }) |
| 899 | } |
| 900 | |
| 901 | gated := func(cx *claimCase) func(r *http.Request) error { |
| 902 | return func(r *http.Request) error { |
| 903 | acct := &Account{ID: OpsAccountID, Sqid: "bab"} |
| 904 | return cx.srv.claimAppMutation(r.Context(), acct, "titogo", "claim-witness", func(ctx context.Context) error { |
| 905 | _, err := cx.srv.cfg.Releases.Put(ctx, appKey, []byte(`{"gated":true}`), "") |
| 906 | return err |
| 907 | }) |
| 908 | } |
| 909 | } |
| 910 | t.Run("admitted passes, foreign closed refuses", func(t *testing.T) { |
| 911 | problems, writers := runClaimDoor(t, claimWitnessDoor("/claim-witness/gated", gated, true)) |
| 912 | if len(problems) != 0 { |
| 913 | t.Fatalf("a door that holds the gate failed the invariant: %v", problems) |
| 914 | } |
| 915 | if !slices.Contains(writers, "bucket/Put") { |
| 916 | t.Fatalf("the gated witness reached writers %v, want bucket/Put", writers) |
| 917 | } |
| 918 | }) |
| 919 | t.Run("an unauthenticated request is not coverage", func(t *testing.T) { |
| 920 | problems, _ := runClaimDoor(t, claimWitnessDoor("/claim-witness/unauthenticated", gated, false)) |
| 921 | if !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, "answered 401") }) || |
| 922 | !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, "reached no raw app effect") }) { |
| 923 | t.Fatalf("a 401 counted as coverage: %v", problems) |
| 924 | } |
| 925 | }) |
| 926 | |
| 927 | // The refused mode: only the owner's real request answered with the |
| 928 | // exact unsupported refusal and no effect at all passes. |
| 929 | refusal := func(path string, authed bool, write func(cx *claimCase) func(r *http.Request) error) *claimRefusalFixture { |
| 930 | return &claimRefusalFixture{ |
| 931 | apps: claimWitnessApps, |
| 932 | setup: func(t *testing.T, cx *claimCase) { |
| 933 | var w func(r *http.Request) error |
| 934 | if write != nil { |
| 935 | w = write(cx) |
| 936 | } |
| 937 | cx.srv.registerRoute("POST "+path, claimWitnessRefusalHandler(cx, w)) |
| 938 | }, |
| 939 | do: func(t *testing.T, cx *claimCase) (int, string) { |
| 940 | token := "" |
| 941 | if authed { |
| 942 | token = cx.bearer |
| 943 | } |
| 944 | status, _, out := rawReq(t, http.MethodPost, cx.ts.URL+path, token, "{}") |
| 945 | return status, out |
| 946 | }, |
| 947 | status: http.StatusConflict, |
| 948 | msg: claimMoveUnsupportedMsg, |
| 949 | } |
| 950 | } |
| 951 | t.Run("an exact authenticated refusal passes", func(t *testing.T) { |
| 952 | if problems := runClaimRefusal(t, refusal("/claim-witness/refused", true, nil)); len(problems) != 0 { |
| 953 | t.Fatalf("an exact unsupported refusal with no effect failed: %v", problems) |
| 954 | } |
| 955 | }) |
| 956 | t.Run("an auth refusal is not an explicit refusal", func(t *testing.T) { |
| 957 | problems := runClaimRefusal(t, refusal("/claim-witness/refused-unauthenticated", false, nil)) |
| 958 | has := func(s string) bool { |
| 959 | return slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, s) }) |
| 960 | } |
| 961 | if !has("open run: the authenticated valid request answered 401") || !has("is not exactly the unsupported refusal") { |
| 962 | t.Fatalf("a 401 counted as an explicit refusal: %v", problems) |
| 963 | } |
| 964 | }) |
| 965 | t.Run("a refusal that reached an effect is not a refusal", func(t *testing.T) { |
| 966 | problems := runClaimRefusal(t, refusal("/claim-witness/refused-after-write", true, gated)) |
| 967 | if !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, "open run: the refusal wrote control object") }) || |
| 968 | !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, "open run: raw bucket Put "+appKey+" is app-scoped") }) { |
| 969 | t.Fatalf("a refusal that wrote passed: %v", problems) |
| 970 | } |
| 971 | }) |
| 972 | t.Run("the census admits only exact unsupported refusals", func(t *testing.T) { |
| 973 | do := func(*testing.T, *claimCase) (int, string) { return 0, "" } |
| 974 | for name, c := range map[string]struct { |
| 975 | f claimRefusalFixture |
| 976 | want string |
| 977 | }{ |
| 978 | "401": {claimRefusalFixture{apps: claimWitnessApps, do: do, status: http.StatusUnauthorized, msg: claimMoveUnsupportedMsg}, "answering 401"}, |
| 979 | "404": {claimRefusalFixture{apps: claimWitnessApps, do: do, status: http.StatusNotFound, msg: claimMoveUnsupportedMsg}, "answering 404"}, |
| 980 | "generic 409": {claimRefusalFixture{apps: claimWitnessApps, do: do, status: http.StatusConflict, msg: claimRetireBusyMsg}, "not an intentionally unsupported operation"}, |
| 981 | "no request": {claimRefusalFixture{apps: claimWitnessApps, status: http.StatusConflict, msg: claimMoveUnsupportedMsg}, "no affected app or no request"}, |
| 982 | } { |
| 983 | f := c.f |
| 984 | problems := claimFixtureCensus(nil, map[string]claimFixture{"route POST /claim-witness/refused": {refused: &f}}) |
| 985 | if !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, c.want) }) { |
| 986 | t.Errorf("%s: the census admitted the refusal: %v", name, problems) |
| 987 | } |
| 988 | } |
| 989 | }) |
| 990 | |
| 991 | // The mutation check for the token-revoke census entry. The real door |
| 992 | // (claimLegacyAppTokenRevokeDoor) passes in TestClaimConsoleMutationInvariant; |
| 993 | // here the same staging and the same valid request are aimed at a route |
| 994 | // that revokes the same token WITHOUT admission, which is exactly what |
| 995 | // RevokeCLIToken did before it was admitted. The census must fail it. |
| 996 | t.Run("the SQL trigger refuses a raw token revoke", func(t *testing.T) { |
| 997 | const path = "/claim-witness/ungated-token-revoke" |
| 998 | d := claimLegacyAppTokenRevokeDoor(path) |
| 999 | stage := d.setup |
| 1000 | d.setup = func(t *testing.T, cx *claimCase) { |
| 1001 | stage(t, cx) |
| 1002 | cx.srv.registerRoute("POST "+path, claimWitnessHandler(cx, func(r *http.Request) error { |
| 1003 | return cx.st.claimRawRevokeCLIToken(claimFxOwner, cx.vars["apptok"]) |
| 1004 | })) |
| 1005 | } |
| 1006 | d.do = func(t *testing.T, cx *claimCase) int { |
| 1007 | status, _, _ := rawReq(t, http.MethodPost, cx.ts.URL+path, cx.bearer, "{}") |
| 1008 | return status |
| 1009 | } |
| 1010 | problems, _ := runClaimDoor(t, d) |
| 1011 | has := func(s string) bool { |
| 1012 | return slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, s) }) |
| 1013 | } |
| 1014 | if !has("never registered with the gate") || !has("still live after the open run") { |
| 1015 | t.Fatalf("an ungated legacy app token revoke passed the census: %v", problems) |
| 1016 | } |
| 1017 | }) |
| 1018 | t.Run("an ungated token-revoke route fails the census", func(t *testing.T) { |
| 1019 | const path = "/claim-witness/ungated-token-revoke" |
| 1020 | d := claimLegacyAppTokenRevokeDoor(path) |
| 1021 | stage := d.setup |
| 1022 | d.setup = func(t *testing.T, cx *claimCase) { |
| 1023 | stage(t, cx) |
| 1024 | cx.srv.registerRoute("POST "+path, claimWitnessHandler(cx, func(r *http.Request) error { |
| 1025 | // Deliberately simulate a forgotten gate that reaches raw SQL. |
| 1026 | // This test-only bypass proves the census detects a LANDED |
| 1027 | // mutation independently of the production SQL trigger. |
| 1028 | scope := []claimSQLScope{{AccountID: OpsAccountID, App: "kass", AppID: cx.appIDs[claimAppRef{Account: "bab", App: "kass"}], Mode: claimModeEffect}} |
| 1029 | return cx.st.withClaimSQL(gateCtx(t), scope, allowAllClaimSQL, func(st *Store) error { return st.claimRawRevokeCLIToken(claimFxOwner, cx.vars["apptok"]) }) |
| 1030 | })) |
| 1031 | } |
| 1032 | d.do = func(t *testing.T, cx *claimCase) int { |
| 1033 | status, _, _ := rawReq(t, http.MethodPost, cx.ts.URL+path, cx.bearer, "{}") |
| 1034 | return status |
| 1035 | } |
| 1036 | problems, _ := runClaimDoor(t, d) |
| 1037 | has := func(s string) bool { |
| 1038 | return slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, s) }) |
| 1039 | } |
| 1040 | if !has("never registered with the gate") || !has("revoked while its app was closed") || !has("SQL rows of bab/kass changed in [cli_tokens]") { |
| 1041 | t.Fatalf("an ungated legacy app token revoke passed the census: %v", problems) |
| 1042 | } |
| 1043 | }) |
| 1044 | t.Run("a rejected token-revoke probe is not coverage", func(t *testing.T) { |
| 1045 | // The signed-in owner with a valid CSRF token, but an id that names |
| 1046 | // no row: the handler answers a redirect (303, below 400) and |
| 1047 | // revokes nothing. Status alone would pass it. |
| 1048 | d := claimLegacyAppTokenRevokeDoor("/auth/cli/tokens/revoke") |
| 1049 | d.do = claimForm("/auth/cli/tokens/revoke", "id", "no-such-listing-id") |
| 1050 | problems, _ := runClaimDoor(t, d) |
| 1051 | has := func(s string) bool { |
| 1052 | return slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, s) }) |
| 1053 | } |
| 1054 | if !has("never registered with the gate") || !has("reached no raw app effect") || !has("still live after the open run") { |
| 1055 | t.Fatalf("a rejected probe counted as coverage: %v", problems) |
| 1056 | } |
| 1057 | }) |
| 1058 | t.Run("a person token revoke needs no admission", func(t *testing.T) { |
| 1059 | cx := newClaimCase(t) |
| 1060 | raw := mintUserToken(t, cx.st, claimFxOwner, time.Now()) |
| 1061 | id := hashToken(raw) |
| 1062 | restore := cx.foreign(t, claimAppRef{Account: "bab", App: "kass"}) |
| 1063 | defer restore() |
| 1064 | w := cx.measureRun(t, claimWitnessApps, func() int { return cx.form(t, "/auth/cli/tokens/revoke", "id", id) }) |
| 1065 | if w.status == 0 || w.status >= 400 { |
| 1066 | t.Fatalf("the person token revoke answered %d", w.status) |
| 1067 | } |
| 1068 | toks, err := cx.st.CLITokens(claimFxOwner) |
| 1069 | if err != nil { |
| 1070 | t.Fatal(err) |
| 1071 | } |
| 1072 | for _, tok := range toks { |
| 1073 | if tok.ID == id { |
| 1074 | t.Fatal("the person token is still live: its revoke needed an admission it must not need") |
| 1075 | } |
| 1076 | } |
| 1077 | }) |
| 1078 | |
| 1079 | // RevokeCLITokenValue deletes by hash alone, so it removes a legacy app |
| 1080 | // token as readily as a person's. The logout door in front of it only |
| 1081 | // admits person tokens (cliUser), but the method does not rely on that: a |
| 1082 | // row that names an app is admitted for that app first. |
| 1083 | t.Run("an app token revoked by its own value is admitted", func(t *testing.T) { |
| 1084 | cx := newClaimCase(t) |
| 1085 | raw, err := cx.st.NewAppToken(OpsAccountID, "kass", claimFxOwner, "census app token", time.Now()) |
| 1086 | if err != nil { |
| 1087 | t.Fatal(err) |
| 1088 | } |
| 1089 | live := func() bool { |
| 1090 | _, ok := cx.st.CLIPrincipal(raw, time.Now()) |
| 1091 | return ok |
| 1092 | } |
| 1093 | if !live() { |
| 1094 | t.Fatal("the staged app token is not live") |
| 1095 | } |
| 1096 | restore := cx.foreign(t, claimAppRef{Account: "bab", App: "kass"}) |
| 1097 | err = cx.st.RevokeCLITokenValueContext(context.Background(), raw) |
| 1098 | restore() |
| 1099 | if err == nil { |
| 1100 | t.Fatal("an app token was revoked by value while its app was closed under a foreign exclusive operation") |
| 1101 | } |
| 1102 | if !live() { |
| 1103 | t.Fatal("the refused revoke still removed the app token's row") |
| 1104 | } |
| 1105 | if err := cx.st.RevokeCLITokenValueContext(context.Background(), raw); err != nil { |
| 1106 | t.Fatalf("the open revoke by value: %v", err) |
| 1107 | } |
| 1108 | if live() { |
| 1109 | t.Fatal("the app token is still live after the open revoke by value") |
| 1110 | } |
| 1111 | if err := cx.st.RevokeCLITokenValueContext(context.Background(), raw); err != nil { |
| 1112 | t.Fatalf("revoking a token that is already gone is not an error: %v", err) |
| 1113 | } |
| 1114 | }) |
| 1115 | t.Run("a person token revoked by value needs no admission", func(t *testing.T) { |
| 1116 | cx := newClaimCase(t) |
| 1117 | raw := mintUserToken(t, cx.st, claimFxOwner, time.Now()) |
| 1118 | restore := cx.foreign(t, claimAppRef{Account: "bab", App: "kass"}) |
| 1119 | defer restore() |
| 1120 | if err := cx.st.RevokeCLITokenValueContext(context.Background(), raw); err != nil { |
| 1121 | t.Fatalf("a person's logout needed an app admission: %v", err) |
| 1122 | } |
| 1123 | if _, ok := cx.st.CLITokenAddress(raw, time.Now()); ok { |
| 1124 | t.Fatal("the person token is still live after its own revoke") |
| 1125 | } |
| 1126 | }) |
| 1127 | |
| 1128 | t.Run("the foreign detector sees each kind of leak", func(t *testing.T) { |
| 1129 | ref := claimAppRef{Account: "bab", App: "titogo"} |
| 1130 | refs := []claimAppRef{ref} |
| 1131 | for name, w := range map[string]claimWindow{ |
| 1132 | "bucket": {events: []claimRawEvent{{Surface: "bucket", Method: "Put", Key: appKey, Scopes: []hostops.AppScope{ref.scope()}}}}, |
| 1133 | "admission": {events: []claimRawEvent{{Surface: "bucket", Method: "Put", Key: admissionKeyOf("bab", "titogo"), Control: true, Scopes: []hostops.AppScope{ref.scope()}}}}, |
| 1134 | "http": {events: []claimRawEvent{{Surface: "http", Method: "POST", Key: "https://app.example/mail/events", Unscoped: true}}}, |
| 1135 | "backend": {events: []claimRawEvent{{Surface: "objstore.Provisioner", Method: "EnsureBucket", Key: "carlos-bab-titogo", Unscoped: true}}}, |
| 1136 | "versioned": {events: []claimRawEvent{{Surface: "versioned:data", Method: "DeleteVersion", Key: "instances/acme.go.tito.io/db", Unscoped: true}}}, |
| 1137 | "sql": {sqlChanged: map[claimAppRef][]string{ref: {"audit"}}}, |
| 1138 | } { |
| 1139 | if len(claimForeignProblems(ref, refs, w)) == 0 { |
| 1140 | t.Errorf("a %s leak under a foreign exclusive went unreported", name) |
| 1141 | } |
| 1142 | } |
| 1143 | quiet := claimWindow{status: http.StatusConflict, events: []claimRawEvent{ |
| 1144 | {Surface: "bucket", Method: "Put", Key: hostops.MailGateKey, Control: true}, |
| 1145 | {Surface: "bucket", Method: "Put", Key: "status/boxes/box.json", NonApp: "deployment status"}, |
| 1146 | }} |
| 1147 | if p := claimForeignProblems(ref, refs, quiet); len(p) != 0 { |
| 1148 | t.Errorf("non-app effects were reported as app leaks: %v", p) |
| 1149 | } |
| 1150 | if p := claimOpenProblems(refs, quiet); len(p) == 0 { |
| 1151 | t.Error("an open run with no admission write and no app effect passed") |
| 1152 | } |
| 1153 | }) |
| 1154 | } |
| 1155 | |
| 1156 | func TestClaimLegacyAppTokenRevokeRegression(t *testing.T) { |
| 1157 | problems, _ := runClaimDoor(t, claimLegacyAppTokenRevokeDoor("/auth/cli/tokens/revoke")) |
| 1158 | for _, problem := range problems { |
| 1159 | t.Error(problem) |
| 1160 | } |
| 1161 | } |
| 1162 | |