| 1 | package console |
| 2 | |
| 3 | // claim_invariant_probes_world_test.go stages the worlds the probes need and |
| 4 | // reads the world back, so a probe proves what a handler DID rather than what |
| 5 | // it answered. Everything here writes below the console (raw bucket, store |
| 6 | // handle, registry) or drives a door the way a client does. |
| 7 | |
| 8 | import ( |
| 9 | "bytes" |
| 10 | "context" |
| 11 | "crypto/sha256" |
| 12 | "encoding/json" |
| 13 | "fmt" |
| 14 | "io" |
| 15 | "mime/multipart" |
| 16 | "net/http" |
| 17 | "net/url" |
| 18 | "testing" |
| 19 | "time" |
| 20 | |
| 21 | "github.com/carlosframework/platform/internal/dbdoor" |
| 22 | "github.com/carlosframework/platform/internal/hostactors" |
| 23 | "github.com/carlosframework/platform/internal/hostclaim" |
| 24 | "github.com/carlosframework/platform/internal/hostfence" |
| 25 | "github.com/carlosframework/platform/internal/hostresolve" |
| 26 | "github.com/carlosframework/platform/internal/objstore" |
| 27 | "github.com/carlosframework/platform/internal/registry" |
| 28 | "github.com/carlosframework/platform/internal/release" |
| 29 | "github.com/carlosframework/platform/internal/status" |
| 30 | ) |
| 31 | |
| 32 | // bucketJSON decodes the object at key from the raw bucket into v. It answers |
| 33 | // false when the object is absent or is not that shape. |
| 34 | func (cx *claimCase) bucketJSON(key string, v any) bool { |
| 35 | raw, _, err := cx.rel.Get(context.Background(), key) |
| 36 | if err != nil { |
| 37 | return false |
| 38 | } |
| 39 | return json.Unmarshal(raw, v) == nil |
| 40 | } |
| 41 | |
| 42 | // fleetBox reads one fleet box straight from the registry file. |
| 43 | func (cx *claimCase) fleetBox(t *testing.T, fleet, label string) (registry.FleetBox, bool) { |
| 44 | t.Helper() |
| 45 | reg, err := registry.Open(cx.regPath) |
| 46 | if err != nil { |
| 47 | t.Fatal(err) |
| 48 | } |
| 49 | boxes, err := reg.ListFleetBoxes(fleet) |
| 50 | if err != nil { |
| 51 | t.Fatal(err) |
| 52 | } |
| 53 | for _, b := range boxes { |
| 54 | if b.Label == label { |
| 55 | return b, true |
| 56 | } |
| 57 | } |
| 58 | return registry.FleetBox{}, false |
| 59 | } |
| 60 | |
| 61 | // fleetNamed reports whether the registry holds a fleet of that name. |
| 62 | func (cx *claimCase) fleetNamed(t *testing.T, name string) bool { |
| 63 | t.Helper() |
| 64 | reg, err := registry.Open(cx.regPath) |
| 65 | if err != nil { |
| 66 | t.Fatal(err) |
| 67 | } |
| 68 | fleets, err := reg.ListFleets() |
| 69 | if err != nil { |
| 70 | t.Fatal(err) |
| 71 | } |
| 72 | for _, f := range fleets { |
| 73 | if f.Name == name { |
| 74 | return true |
| 75 | } |
| 76 | } |
| 77 | return false |
| 78 | } |
| 79 | |
| 80 | // fleetBoxWorld is fleet pi with one active box, pi-1; the box's token hash is |
| 81 | // kept in vars so a rotation can be seen to change it. |
| 82 | func (cx *claimCase) fleetBoxWorld(t *testing.T) { |
| 83 | t.Helper() |
| 84 | cx.fleet(t) |
| 85 | cx.mustAPI(t, http.MethodPost, "/api/cli/fleets/bab/pi/boxes", `{"label":"pi-1"}`, http.StatusOK) |
| 86 | box, ok := cx.fleetBox(t, "pi", "pi-1") |
| 87 | if !ok { |
| 88 | t.Fatal("stage fleet box: pi/pi-1 is not in the registry") |
| 89 | } |
| 90 | cx.vars["boxhash"] = box.TokenHash |
| 91 | } |
| 92 | |
| 93 | // entitlement reads bab's storage entitlement from the bucket. |
| 94 | func (cx *claimCase) entitlement(t *testing.T) *objstore.Entitlement { |
| 95 | t.Helper() |
| 96 | e, _, err := objstore.GetEntitlement(context.Background(), cx.rel, "bab") |
| 97 | if err != nil { |
| 98 | t.Fatal(err) |
| 99 | } |
| 100 | return e |
| 101 | } |
| 102 | |
| 103 | // accountNamed reports whether address belongs to an account of that name. |
| 104 | func (cx *claimCase) accountNamed(t *testing.T, address, name string) bool { |
| 105 | t.Helper() |
| 106 | accts, err := cx.st.AccountsFor(address) |
| 107 | if err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | for _, a := range accts { |
| 111 | if a.Name == name { |
| 112 | return true |
| 113 | } |
| 114 | } |
| 115 | return false |
| 116 | } |
| 117 | |
| 118 | // mintedAccount is an empty account the owner owns, minted the way the console |
| 119 | // mints one (the bucket manifest and the owner's membership), and its sqid. |
| 120 | func (cx *claimCase) mintedAccount(t *testing.T, name string) string { |
| 121 | t.Helper() |
| 122 | a, err := cx.srv.mintAccount(context.Background(), name, claimFxOwner) |
| 123 | if err != nil { |
| 124 | t.Fatalf("mint account %s: %v", name, err) |
| 125 | } |
| 126 | return a.Sqid |
| 127 | } |
| 128 | |
| 129 | // cookieOf reads one cookie a browser holds for the console. |
| 130 | func (cx *claimCase) cookieOf(c *http.Client, name string) string { |
| 131 | u, err := url.Parse(cx.ts.URL) |
| 132 | if err != nil { |
| 133 | return "" |
| 134 | } |
| 135 | for _, ck := range c.Jar.Cookies(u) { |
| 136 | if ck.Name == name { |
| 137 | return ck.Value |
| 138 | } |
| 139 | } |
| 140 | return "" |
| 141 | } |
| 142 | |
| 143 | // signedInAs answers whether browser c holds a live console session for |
| 144 | // address. |
| 145 | func (cx *claimCase) signedInAs(c *http.Client, address string) bool { |
| 146 | raw := cx.cookieOf(c, sessionCookie) |
| 147 | if raw == "" { |
| 148 | return false |
| 149 | } |
| 150 | got, ok := cx.st.SessionAddress(raw, time.Now()) |
| 151 | return ok && got == address |
| 152 | } |
| 153 | |
| 154 | // signInReply walks the OAuth ceremony on browser c and answers the CALLBACK, |
| 155 | // the request under test (signUp does the same walk and asserts instead). |
| 156 | func (cx *claimCase) signInReply(t *testing.T, c *http.Client, code string) claimReply { |
| 157 | t.Helper() |
| 158 | status, loc, _ := get(t, c, cx.ts.URL+"/auth/signin") |
| 159 | if status != http.StatusSeeOther { |
| 160 | t.Fatalf("the sign-in start answered %d, want 303", status) |
| 161 | } |
| 162 | u, err := url.Parse(loc) |
| 163 | if err != nil { |
| 164 | t.Fatal(err) |
| 165 | } |
| 166 | q := u.Query() |
| 167 | cx.fk.mu.Lock() |
| 168 | cx.fk.challenge = q.Get("code_challenge") |
| 169 | cx.fk.expectCode = code |
| 170 | cx.fk.mu.Unlock() |
| 171 | status, loc, body := get(t, c, cx.ts.URL+"/auth/callback?code="+url.QueryEscape(code)+"&state="+url.QueryEscape(q.Get("state"))) |
| 172 | return cx.browserReply(c, status, loc, body) |
| 173 | } |
| 174 | |
| 175 | // ---- the platform system door --------------------------------------------------- |
| 176 | |
| 177 | // systemUploadReply parks one local build (version, linux-arm64) through the |
| 178 | // upload door. |
| 179 | func (cx *claimCase) systemUploadReply(t *testing.T, version string) claimReply { |
| 180 | t.Helper() |
| 181 | var buf bytes.Buffer |
| 182 | mw := multipart.NewWriter(&buf) |
| 183 | meta, err := mw.CreateFormField("meta") |
| 184 | if err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | if _, err := meta.Write([]byte(`{"version":"` + version + `","platform":"linux-arm64"}`)); err != nil { |
| 188 | t.Fatal(err) |
| 189 | } |
| 190 | file, err := mw.CreateFormFile("file", "carlos-linux-arm64") |
| 191 | if err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | if _, err := file.Write([]byte("ELF " + version)); err != nil { |
| 195 | t.Fatal(err) |
| 196 | } |
| 197 | if err := mw.Close(); err != nil { |
| 198 | t.Fatal(err) |
| 199 | } |
| 200 | req, err := http.NewRequest(http.MethodPut, cx.ts.URL+"/api/cli/system/upload", &buf) |
| 201 | if err != nil { |
| 202 | t.Fatal(err) |
| 203 | } |
| 204 | req.Header.Set("Content-Type", mw.FormDataContentType()) |
| 205 | req.Header.Set("Authorization", "Bearer "+cx.bearer) |
| 206 | res, err := http.DefaultClient.Do(req) |
| 207 | if err != nil { |
| 208 | t.Fatal(err) |
| 209 | } |
| 210 | defer res.Body.Close() |
| 211 | out, err := io.ReadAll(res.Body) |
| 212 | if err != nil { |
| 213 | t.Fatal(err) |
| 214 | } |
| 215 | return claimReply{Status: res.StatusCode, Body: string(out)} |
| 216 | } |
| 217 | |
| 218 | // systemJob polls one platform job until it is done and answers its record. |
| 219 | func (cx *claimCase) systemJob(t *testing.T, id string) map[string]any { |
| 220 | t.Helper() |
| 221 | for i := 0; i < 200; i++ { |
| 222 | r := cx.apiReply(t, http.MethodGet, "/api/cli/system/jobs/"+id, "") |
| 223 | if r.Status != http.StatusOK { |
| 224 | t.Fatalf("job %s answered %s", id, r) |
| 225 | } |
| 226 | var job map[string]any |
| 227 | if err := json.Unmarshal([]byte(r.Body), &job); err != nil { |
| 228 | t.Fatalf("job %s: %v", id, err) |
| 229 | } |
| 230 | if job["done"] == true { |
| 231 | return job |
| 232 | } |
| 233 | time.Sleep(20 * time.Millisecond) |
| 234 | } |
| 235 | t.Fatalf("job %s never finished", id) |
| 236 | return nil |
| 237 | } |
| 238 | |
| 239 | // systemRelease promotes one uploaded local build to a channel, through the |
| 240 | // upload and update doors, and waits for the job. |
| 241 | func (cx *claimCase) systemRelease(t *testing.T, channel, version string) string { |
| 242 | t.Helper() |
| 243 | cx.stageSystemCandidate(t, version) |
| 244 | up := cx.systemUploadReply(t, version) |
| 245 | if up.Status != http.StatusOK { |
| 246 | t.Fatalf("stage upload %s: %s", version, up) |
| 247 | } |
| 248 | handle := claimReplyField(t, up.Body, "upload") |
| 249 | out := cx.mustAPI(t, http.MethodPost, "/api/cli/system/update", fmt.Sprintf(`{"channel":%q,"upload":%q}`, channel, handle), http.StatusOK) |
| 250 | id := claimReplyField(t, out, "job") |
| 251 | if job := cx.systemJob(t, id); job["error"] != "" { |
| 252 | t.Fatalf("stage update %s: job failed: %v", version, job) |
| 253 | } |
| 254 | return id |
| 255 | } |
| 256 | |
| 257 | // systemVersion is the version a channel's pointer names now, "" when it names |
| 258 | // none. |
| 259 | func (cx *claimCase) systemVersion(channel string) string { |
| 260 | p, _, err := release.GetSystemPointer(context.Background(), cx.rel, cx.pub, release.SystemComponentRouter, channel) |
| 261 | if err != nil || p == nil { |
| 262 | return "" |
| 263 | } |
| 264 | return p.Current.Version |
| 265 | } |
| 266 | |
| 267 | // ---- the replica door ------------------------------------------------------------ |
| 268 | |
| 269 | const ( |
| 270 | claimDBHost = "acme.go.tito.io" |
| 271 | claimDBBucket = "data-claim" |
| 272 | ) |
| 273 | |
| 274 | // dbWorld is the `carlos db` door's world on an enabled console: a real |
| 275 | // instance with its host claim, this console serving its region, a data bucket |
| 276 | // and one level-0 LTX file in the instance's replica location. |
| 277 | func (cx *claimCase) dbWorld(t *testing.T) { |
| 278 | t.Helper() |
| 279 | cx.movableInstance(t) |
| 280 | cx.srv.cfg.Region = "ie" |
| 281 | cx.srv.cfg.HibernateBucket = claimDBBucket |
| 282 | cx.versions.AddBucket(claimDBBucket) |
| 283 | obs, err := hostclaim.Read(context.Background(), cx.rel, claimDBHost) |
| 284 | if err != nil || obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || obs.Claim.Holder.Location == "" { |
| 285 | t.Fatalf("stage the db world: %s has no claim with a replica location: %+v %v", claimDBHost, obs, err) |
| 286 | } |
| 287 | cx.versions.Put(claimDBBucket, dbVersionKey(obs.Claim.Holder.Location, 0, 1, 1), doorLTXHeader(t, 1, 1)) |
| 288 | } |
| 289 | |
| 290 | // dbHandle lists the replica as the owner and answers the signed handle of the |
| 291 | // level-0 file, which is what a file download must present. |
| 292 | func (cx *claimCase) dbHandle(t *testing.T) string { |
| 293 | t.Helper() |
| 294 | r := cx.apiReply(t, http.MethodGet, dbdoor.ListPath("bab", "titogo", claimDBHost, 0), "") |
| 295 | if r.Status != http.StatusOK { |
| 296 | t.Fatalf("stage the db handle: the listing answered %s", r) |
| 297 | } |
| 298 | var list dbdoor.ListResponse |
| 299 | if err := json.Unmarshal([]byte(r.Body), &list); err != nil { |
| 300 | t.Fatalf("stage the db handle: %v: %s", err, r) |
| 301 | } |
| 302 | for _, f := range list.Files { |
| 303 | if f.Level == 0 && f.MinTXID == 1 && f.MaxTXID == 1 && f.Handle != "" { |
| 304 | return f.Handle |
| 305 | } |
| 306 | } |
| 307 | t.Fatalf("stage the db handle: the listing names no level-0 file 1-1: %s", r) |
| 308 | return "" |
| 309 | } |
| 310 | |
| 311 | // apiReplyWith sends one CLI request as the owner with extra headers. |
| 312 | func (cx *claimCase) apiReplyWith(t *testing.T, method, path string, header map[string]string) claimReply { |
| 313 | t.Helper() |
| 314 | req, err := http.NewRequest(method, cx.ts.URL+cx.expand(path), nil) |
| 315 | if err != nil { |
| 316 | t.Fatal(err) |
| 317 | } |
| 318 | req.Header.Set("Authorization", "Bearer "+cx.bearer) |
| 319 | for k, v := range header { |
| 320 | req.Header.Set(k, v) |
| 321 | } |
| 322 | res, err := http.DefaultClient.Do(req) |
| 323 | if err != nil { |
| 324 | t.Fatal(err) |
| 325 | } |
| 326 | defer res.Body.Close() |
| 327 | out, err := io.ReadAll(res.Body) |
| 328 | if err != nil { |
| 329 | t.Fatal(err) |
| 330 | } |
| 331 | return claimReply{Status: res.StatusCode, Loc: res.Header.Get("Location"), Body: string(out)} |
| 332 | } |
| 333 | |
| 334 | // ---- host loss ------------------------------------------------------------------ |
| 335 | |
| 336 | const claimLossHost = "shop.bab.oncarlos.com" |
| 337 | |
| 338 | // claimLossBox is the roster box the operator declares lost (it is the world's |
| 339 | // own tito box, so the roster needs no edit). |
| 340 | const claimLossBox = "box-ie-tito" |
| 341 | |
| 342 | // hostLossWorld stages one serving claim on claimLossHost whose other box is |
| 343 | // claimLossBox: the shape a lost box leaves behind. It writes the raw bucket, |
| 344 | // below every boundary, as the actor simulator does. |
| 345 | func (cx *claimCase) hostLossWorld(t *testing.T) { |
| 346 | t.Helper() |
| 347 | ctx := context.Background() |
| 348 | obs, err := hostclaim.Read(ctx, cx.rel, claimLossHost) |
| 349 | if err != nil { |
| 350 | t.Fatal(err) |
| 351 | } |
| 352 | n := 0 |
| 353 | nh := hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + claimLossHost + ".json", Account: "bab", App: "jam", |
| 354 | Host: claimLossHost, Region: "ie", RecordRegion: "ie"} |
| 355 | if _, err := hostclaim.Update(ctx, cx.rel, obs, hostclaim.Adopt(nh, "box-ie-default", nil, []string{claimLossBox}, "discovery", |
| 356 | func() string { n++; return fmt.Sprintf("claim-loss-%d", n) })); err != nil { |
| 357 | t.Fatalf("stage the serving claim of %s: %v", claimLossHost, err) |
| 358 | } |
| 359 | } |
| 360 | |
| 361 | // lossInspection asks the inspect door for the challenge and decodes it. |
| 362 | func (cx *claimCase) lossInspection(t *testing.T) hostresolve.Inspection { |
| 363 | t.Helper() |
| 364 | r := cx.apiReply(t, http.MethodPost, "/api/cli/ops/hosts/"+claimLossHost+"/resolve/inspect", |
| 365 | `{"action":"declare-lost","box":"`+claimLossBox+`"}`) |
| 366 | if r.Status != http.StatusOK { |
| 367 | t.Fatalf("inspect %s: %s", claimLossHost, r) |
| 368 | } |
| 369 | var in hostresolve.Inspection |
| 370 | if err := json.Unmarshal([]byte(r.Body), &in); err != nil || len(in.Challenge) == 0 || in.Details.ID == "" { |
| 371 | t.Fatalf("inspect %s answered no challenge: %v %s", claimLossHost, err, r) |
| 372 | } |
| 373 | return in |
| 374 | } |
| 375 | |
| 376 | // lossSubmission is the operator's side of a declaration: the cloud |
| 377 | // observation the challenge asks for and the attestations it names. |
| 378 | func claimLossSubmission(in hostresolve.Inspection) hostfence.Submission { |
| 379 | return hostfence.Submission{ |
| 380 | Challenge: in.Challenge, |
| 381 | Confirmation: in.Details.ID, |
| 382 | Observation: hostfence.CloudObservation{ |
| 383 | ObservedAt: in.Details.IssuedAt.Add(time.Millisecond), Account: in.Details.Cloud.Account, |
| 384 | Region: in.Details.Cloud.Region, InstanceID: in.Details.Cloud.InstanceID, State: "stopped", RequestID: "claim-fixture-request"}, |
| 385 | Attestation: hostfence.OperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true, |
| 386 | Disks: in.Details.Consequences.DestroyedStorageAssertion}, |
| 387 | } |
| 388 | } |
| 389 | |
| 390 | // rosterBox reads one roster box through the deployment directory. |
| 391 | func (cx *claimCase) rosterBox(t *testing.T, id string) hostactors.Box { |
| 392 | t.Helper() |
| 393 | d := hostactors.Directory{Store: cx.rel, PublicKey: cx.pub, Bucket: claimFxBucket} |
| 394 | o, err := d.Read(context.Background()) |
| 395 | if err != nil { |
| 396 | t.Fatal(err) |
| 397 | } |
| 398 | return o.Roster.Boxes[id] |
| 399 | } |
| 400 | |
| 401 | // An enabled update requires a release-workflow identity and fresh floor- |
| 402 | // capable box reports. All boxes follow stable; these canary updates have |
| 403 | // no pending box acknowledgements, and still run the real approval checks. |
| 404 | func (cx *claimCase) stageSystemCandidate(t *testing.T, version string) { |
| 405 | t.Helper() |
| 406 | ctx := context.Background() |
| 407 | tag := "v1.1.0" |
| 408 | if version == "77698ab" { |
| 409 | tag = "v1.2.0" |
| 410 | } |
| 411 | sum := sha256.Sum256([]byte("ELF " + version)) |
| 412 | rel := release.SystemRelease{Version: version, ReleaseTag: tag, Source: "release:" + tag, MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{"linux-arm64": {Key: release.SystemArtifactKey(version, "linux-arm64"), SHA256: fmt.Sprintf("%x", sum)}}} |
| 413 | st := release.NewDirStore(t.TempDir()) |
| 414 | sg := release.RootSigner(cx.priv) |
| 415 | if _, err := release.PromoteSystem(ctx, st, sg, "router", "canary", rel, release.SystemPromoteOptions{}); err != nil { |
| 416 | t.Fatal(err) |
| 417 | } |
| 418 | wire, _, err := st.Get(ctx, release.SystemPointerKey("router", "canary")) |
| 419 | if err != nil { |
| 420 | t.Fatal(err) |
| 421 | } |
| 422 | if err = release.PublishSystemIdentity(ctx, cx.rel, cx.pub, wire); err != nil { |
| 423 | t.Fatal(err) |
| 424 | } |
| 425 | d := cx.claimFixtureDirectory() |
| 426 | o, err := d.Read(ctx) |
| 427 | if err != nil { |
| 428 | t.Fatal(err) |
| 429 | } |
| 430 | for ref, b := range o.Roster.Boxes { |
| 431 | report, err := cx.srv.cfg.ClaimAuthority.Evidence.BoxReport(ctx, ref, b) |
| 432 | if err != nil { |
| 433 | t.Fatal(err) |
| 434 | } |
| 435 | box := status.Box{GeneratedAt: report.GeneratedAt, Metrics: &status.BoxMetrics{Enrollment: report.Enrollment, InstanceID: report.InstanceID, Release: report.Release, ReleaseTag: report.ReleaseTag, Platform: report.Platform, BinarySHA256: report.BinarySHA256, MovesProtocol: report.MovesProtocol, BootID: report.BootID, FloorSHA256: report.FloorSHA256, FloorGuardReady: true, HostClaimEnabled: true, SystemChannel: "stable", Layout: &status.LayoutReport{Mode: "flat", Rows: map[string]status.RowLayout{}}}} |
| 436 | body, err := json.Marshal(box) |
| 437 | if err != nil { |
| 438 | t.Fatal(err) |
| 439 | } |
| 440 | key := "status/boxes/" + ref + ".json" |
| 441 | _, etag, _ := cx.rel.Get(ctx, key) |
| 442 | if _, err = cx.rel.Put(ctx, key, body, etag); err != nil { |
| 443 | t.Fatal(err) |
| 444 | } |
| 445 | } |
| 446 | } |
| 447 | |
amadan