| 1 | // Task 5 of the fleet-api plan: the spec's central promise proved end to end |
| 2 | // — a fleet box holding NO bucket credentials converges entirely through the |
| 3 | // store door (spec 2026-08-11-fleet-api-surface-design.md, F1-F8; issue #73). |
| 4 | // |
| 5 | // The box's whole view of the platform's deployment bucket is a |
| 6 | // fleetapi.New(...) client pointed at a REAL edge: real TunnelHub auth |
| 7 | // (registry-backed, the same table detach/rotate converge), the real |
| 8 | // fleet→account snapshot Reload builds, the real ACL, a DirStore behind it. |
| 9 | // No box-side code path in this file ever touches that DirStore — every |
| 10 | // platform-side use of it below is the OTHER role (the console/CLI writing |
| 11 | // the bucket, or the assertion reading back what a box wrote through the |
| 12 | // door). |
| 13 | // |
| 14 | // It sits beside fleet_e2e_test.go rather than inside it because the two |
| 15 | // prove different things about the same fleet and want different worlds: |
| 16 | // TestTwoBoxFleetEndToEnd drives the ROUTING failure sequence (channels, |
| 17 | // leases, takeover, dark fleet) and its edge deliberately has no static |
| 18 | // store — no door at all. This one drives the STORE loop (adopt, serial, |
| 19 | // provision, heartbeat, bindings, logs) and needs EnableStatic wired. What |
| 20 | // they do share is shared: e2eHashToken, startFleetBox, namedFleetHandler, |
| 21 | // waitChannel, e2eGet and waitBody all come from that file, so the edge/hub |
| 22 | // arrangement has exactly one source of truth per concern. |
| 23 | // |
| 24 | // Hermetic and fast: loopback httptest edge, real h2 reverse channels via |
| 25 | // the production tunnelclient.Loop, real ed25519 release signing. |
| 26 | package edge_test |
| 27 | |
| 28 | import ( |
| 29 | "context" |
| 30 | "crypto/ed25519" |
| 31 | "encoding/json" |
| 32 | "errors" |
| 33 | "net/http/httptest" |
| 34 | "net/url" |
| 35 | "os" |
| 36 | "path/filepath" |
| 37 | "runtime" |
| 38 | "slices" |
| 39 | "strings" |
| 40 | "testing" |
| 41 | "time" |
| 42 | |
| 43 | "github.com/carlosframework/platform/internal/applog" |
| 44 | "github.com/carlosframework/platform/internal/domains" |
| 45 | "github.com/carlosframework/platform/internal/edge" |
| 46 | "github.com/carlosframework/platform/internal/fleetapi" |
| 47 | "github.com/carlosframework/platform/internal/host" |
| 48 | "github.com/carlosframework/platform/internal/provision" |
| 49 | "github.com/carlosframework/platform/internal/registry" |
| 50 | "github.com/carlosframework/platform/internal/release" |
| 51 | "github.com/carlosframework/platform/internal/status" |
| 52 | ) |
| 53 | |
| 54 | const ( |
| 55 | fapiFleet = "orchard" |
| 56 | fapiBoxA = "box-a" |
| 57 | fapiBoxB = "box-b" |
| 58 | fapiTokenA = "orchard-token-a" |
| 59 | fapiTokenB = "orchard-token-b" |
| 60 | |
| 61 | fapiApp = "shop" |
| 62 | fapiDomain = "orchard.example" |
| 63 | fapiHost = "acme." + fapiDomain |
| 64 | fapiRegion = "ie" |
| 65 | fapiChannel = "canary/e2e" // canary/* has no bake window: the pass applies at once |
| 66 | fapiVersion = "v1" |
| 67 | |
| 68 | // The two account identities. Both are real sqids (domains.Sqid of an id), |
| 69 | // never literals: provision.knownAccounts refuses any account manifest |
| 70 | // whose path sqid, body sqid and Sqid(body.ID) do not all agree, so a |
| 71 | // hardcoded string would quietly stop being an account the moment the |
| 72 | // frozen encoding changed — and this test's whole provisioning leg would |
| 73 | // then pass for the wrong reason (nothing known, nothing minted). |
| 74 | fapiAcctID int64 = 7 |
| 75 | fapiOtherID int64 = 8 |
| 76 | fapiOtherApp = "intruder" |
| 77 | ) |
| 78 | |
| 79 | // fapiPlatform is the artifact key a box of THIS machine's shape adopts — |
| 80 | // release's own adoptPlatform, which is unexported. |
| 81 | var fapiPlatform = runtime.GOOS + "-" + runtime.GOARCH |
| 82 | |
| 83 | // fleetDoorWorld is one platform + one fleet: the edge and its bucket on one |
| 84 | // side, two fleet boxes whose only bucket access is the door on the other. |
| 85 | type fleetDoorWorld struct { |
| 86 | reg *registry.Registry // the PLATFORM's registry: fleet rows, box credentials, fleet-placed routes |
| 87 | store *release.DirStore // the platform's deployment bucket — edge-side and console-side handle |
| 88 | srvURL string |
| 89 | hub *edge.TunnelHub |
| 90 | priv ed25519.PrivateKey |
| 91 | pub ed25519.PublicKey |
| 92 | sqid string // the fleet's owning account |
| 93 | other string // a second tenant on the same bucket, which the fleet must never see |
| 94 | boxA release.Store |
| 95 | boxB release.Store |
| 96 | } |
| 97 | |
| 98 | // newFleetDoorWorld stands the world up and returns it with both boxes' |
| 99 | // channels connected — the same credential that dials the tunnel is the one |
| 100 | // their store clients authenticate with, which is F1/F2 stated as a fixture. |
| 101 | func newFleetDoorWorld(t *testing.T) *fleetDoorWorld { |
| 102 | t.Helper() |
| 103 | w := &fleetDoorWorld{ |
| 104 | sqid: domains.Sqid(fapiAcctID), |
| 105 | other: domains.Sqid(fapiOtherID), |
| 106 | } |
| 107 | |
| 108 | // Not closed at cleanup, for the reason fleet_e2e_test.go gives: Edge.Watch |
| 109 | // may still be mid-poll when the test returns. |
| 110 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 111 | if err != nil { |
| 112 | t.Fatalf("open registry: %v", err) |
| 113 | } |
| 114 | w.reg = reg |
| 115 | if err := reg.UpsertFleet(registry.Fleet{Name: fapiFleet, Account: w.sqid, Bucket: "orchard-data"}); err != nil { |
| 116 | t.Fatalf("UpsertFleet: %v", err) |
| 117 | } |
| 118 | for label, tok := range map[string]string{fapiBoxA: fapiTokenA, fapiBoxB: fapiTokenB} { |
| 119 | if err := reg.AddFleetBox(registry.FleetBox{ |
| 120 | Fleet: fapiFleet, Label: label, TokenHash: e2eHashToken(tok), Status: "active", |
| 121 | }); err != nil { |
| 122 | t.Fatalf("AddFleetBox(%s): %v", label, err) |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | w.store = release.NewDirStore(t.TempDir()) |
| 127 | seedB64, pubB64, err := release.GenerateKey() |
| 128 | if err != nil { |
| 129 | t.Fatalf("GenerateKey: %v", err) |
| 130 | } |
| 131 | if w.priv, err = release.ParseSigningKey(seedB64, "test"); err != nil { |
| 132 | t.Fatalf("ParseSigningKey: %v", err) |
| 133 | } |
| 134 | if w.pub, err = release.ParsePublicKey(pubB64, "test"); err != nil { |
| 135 | t.Fatalf("ParsePublicKey: %v", err) |
| 136 | } |
| 137 | |
| 138 | e, err := edge.New(reg, nil) |
| 139 | if err != nil { |
| 140 | t.Fatalf("edge.New: %v", err) |
| 141 | } |
| 142 | srv := httptest.NewServer(e) |
| 143 | t.Cleanup(srv.Close) |
| 144 | w.srvURL = srv.URL |
| 145 | u, err := url.Parse(srv.URL) |
| 146 | if err != nil { |
| 147 | t.Fatal(err) |
| 148 | } |
| 149 | // EnableStatic is what hands the door its store handle — the edge's OWN, |
| 150 | // never a second client, and the door is only reachable on the hub's |
| 151 | // reserved hostname (here the listener's own, as in production |
| 152 | // CARLOS_TUNNEL_HOST). |
| 153 | w.hub = edge.NewTunnelHub(u.Hostname(), nil) |
| 154 | e.EnableStatic(w.store, w.pub, filepath.Join(t.TempDir(), "cache")) |
| 155 | e.EnableTunnels(w.hub) |
| 156 | if err := e.Reload(); err != nil { |
| 157 | t.Fatalf("Reload: %v", err) |
| 158 | } |
| 159 | stopWatch := make(chan struct{}) |
| 160 | t.Cleanup(func() { close(stopWatch) }) |
| 161 | go e.Watch(stopWatch) |
| 162 | |
| 163 | stopA := startFleetBox(t, srv.URL, fapiFleet+"/"+fapiBoxA, fapiTokenA, namedFleetHandler(fapiBoxA)) |
| 164 | t.Cleanup(stopA) |
| 165 | stopB := startFleetBox(t, srv.URL, fapiFleet+"/"+fapiBoxB, fapiTokenB, namedFleetHandler(fapiBoxB)) |
| 166 | t.Cleanup(stopB) |
| 167 | waitChannel(t, w.hub, fapiFleet+"/"+fapiBoxA, "connected", e2eDialBound) |
| 168 | waitChannel(t, w.hub, fapiFleet+"/"+fapiBoxB, "connected", e2eDialBound) |
| 169 | |
| 170 | // The boxes' deployment stores. srv.URL is the door's base exactly as |
| 171 | // CARLOS_TUNNEL_URL is in production, and the name is the box's own |
| 172 | // channel identity — no AWS config, no bucket name, no credentials. |
| 173 | w.boxA = fapiStore(t, srv.URL, fapiBoxA, fapiTokenA) |
| 174 | w.boxB = fapiStore(t, srv.URL, fapiBoxB, fapiTokenB) |
| 175 | return w |
| 176 | } |
| 177 | |
| 178 | func fapiStore(t *testing.T, base, label, token string) release.Store { |
| 179 | t.Helper() |
| 180 | s, err := fleetapi.New(base, fapiFleet+"/"+label, token) |
| 181 | if err != nil { |
| 182 | t.Fatalf("fleetapi.New(%s): %v", label, err) |
| 183 | } |
| 184 | return s |
| 185 | } |
| 186 | |
| 187 | // put is a platform-side (console/CLI-shaped) write straight to the bucket: |
| 188 | // create-only, the same shape every real writer of these keys uses. |
| 189 | func (w *fleetDoorWorld) put(t *testing.T, key, body string) { |
| 190 | t.Helper() |
| 191 | if _, err := w.store.Put(context.Background(), key, []byte(body), ""); err != nil { |
| 192 | t.Fatalf("seed %s: %v", key, err) |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | // accountManifest is domains' account record at the key provision reads — |
| 197 | // accounts/<sqid>/account.json, one level deep (never accounts/<sqid>.json). |
| 198 | func fapiAccountManifest(id int64, sqid, name string) string { |
| 199 | b, _ := json.Marshal(map[string]any{ |
| 200 | "id": id, "sqid": sqid, "name": name, "created_at": "2026-08-11T09:00:00Z", |
| 201 | }) |
| 202 | return string(b) |
| 203 | } |
| 204 | |
| 205 | // TestFleetBoxConvergesWithoutBucketCredentials is Task 5's acceptance |
| 206 | // evidence. The legs run in sequence against ONE world, each depending on |
| 207 | // what the last left behind (a record becomes a route, the route adopts a |
| 208 | // release, the box heartbeats about it, its binding lands in routing) — a |
| 209 | // failure means the loop stopped there and nothing after it is meaningful. |
| 210 | func TestFleetBoxConvergesWithoutBucketCredentials(t *testing.T) { |
| 211 | ctx := context.Background() |
| 212 | w := newFleetDoorWorld(t) |
| 213 | |
| 214 | // A second tenant on the same bucket, planted BEFORE anything else so |
| 215 | // every listing below runs against a store that really does hold another |
| 216 | // account's keys — the negative leg (f) is only meaningful if the data it |
| 217 | // says is invisible is genuinely there. |
| 218 | w.put(t, "accounts/"+w.other+"/account.json", fapiAccountManifest(fapiOtherID, w.other, "Intruder")) |
| 219 | otherChannelKey := "apps/" + w.other + "/" + fapiOtherApp + "/channels/stable.json" |
| 220 | w.put(t, otherChannelKey, `{"app":"intruder","version":"v9"}`) |
| 221 | otherLogKey, err := applog.ChunkKey(w.other, fapiOtherApp, fapiBoxA, applog.HourOf(time.Now()), 1, 9) |
| 222 | if err != nil { |
| 223 | t.Fatal(err) |
| 224 | } |
| 225 | w.put(t, otherLogKey, "not yours") |
| 226 | w.put(t, "fleets/other/status/boxes/box-a.json", `{"label":"box-a"}`) |
| 227 | |
| 228 | // --- (a) the 2s convergence poll: control/serial, read-only -------------- |
| 229 | // The one shared platform key a box polls every 2 seconds. It must read, |
| 230 | // and it must never be writable — a box that could bump the serial could |
| 231 | // make every other box in the deployment run a full pass set on demand. |
| 232 | w.put(t, release.SerialKey, "41") |
| 233 | serial, _, err := w.boxA.Get(ctx, release.SerialKey) |
| 234 | if err != nil || string(serial) != "41" { |
| 235 | t.Fatalf("(a) serial through the door = %q, %v; want \"41\"", serial, err) |
| 236 | } |
| 237 | _, etag, err := w.store.Get(ctx, release.SerialKey) |
| 238 | if err != nil { |
| 239 | t.Fatalf("(a) read serial etag: %v", err) |
| 240 | } |
| 241 | if _, err := w.store.Put(ctx, release.SerialKey, []byte("42"), etag); err != nil { |
| 242 | t.Fatalf("(a) console bump: %v", err) |
| 243 | } |
| 244 | if serial, _, err = w.boxA.Get(ctx, release.SerialKey); err != nil || string(serial) != "42" { |
| 245 | t.Fatalf("(a) serial after the console bump = %q, %v; want \"42\" — the poll would never see a deploy", serial, err) |
| 246 | } |
| 247 | if _, err := w.boxA.Put(ctx, release.SerialKey, []byte("999"), ""); err == nil { |
| 248 | t.Fatal("(a) a fleet box bumped control/serial through the door") |
| 249 | } |
| 250 | t.Log("(a) control/serial reads through the door and refuses the box's write") |
| 251 | |
| 252 | // --- (b) provisioning: an instance record becomes a local route ---------- |
| 253 | // The account-scoped LISTING path end to end: knownAccounts lists |
| 254 | // accounts/ (a shared prefix, filtered per tenant), the record scan lists |
| 255 | // apps/ (scope-mapped to this account's subtree), and the pass mints. |
| 256 | w.put(t, "accounts/"+w.sqid+"/account.json", fapiAccountManifest(fapiAcctID, w.sqid, "Orchard")) |
| 257 | if err := provision.PutManifest(ctx, w.store, w.sqid, fapiApp, provision.Manifest{ |
| 258 | Schema: 1, App: fapiApp, Domain: fapiDomain, Regions: []string{fapiRegion}, Seed: []string{"serve"}, |
| 259 | }); err != nil { |
| 260 | t.Fatalf("(b) PutManifest: %v", err) |
| 261 | } |
| 262 | if err := provision.CreateRecord(ctx, w.store, provision.Record{ |
| 263 | Schema: 1, Host: fapiHost, Slug: "acme", App: fapiApp, Account: w.sqid, |
| 264 | Region: fapiRegion, Channel: fapiChannel, CreatedAt: time.Now(), |
| 265 | }); err != nil { |
| 266 | t.Fatalf("(b) CreateRecord: %v", err) |
| 267 | } |
| 268 | // The other tenant's record, in THIS box's region and pool: the only thing |
| 269 | // keeping it off this box is the door's account scope. |
| 270 | if err := provision.CreateRecord(ctx, w.store, provision.Record{ |
| 271 | Schema: 1, Host: "evil." + fapiDomain, Slug: "evil", App: fapiOtherApp, Account: w.other, |
| 272 | Region: fapiRegion, Channel: "stable", CreatedAt: time.Now(), |
| 273 | }); err != nil { |
| 274 | t.Fatalf("(b) CreateRecord (other tenant): %v", err) |
| 275 | } |
| 276 | |
| 277 | // The box's OWN registry — a fleet box has one, separate from the |
| 278 | // platform's, and this pass is what puts routes in it. |
| 279 | boxReg, err := registry.Open(filepath.Join(t.TempDir(), "box-registry.db")) |
| 280 | if err != nil { |
| 281 | t.Fatalf("(b) open box registry: %v", err) |
| 282 | } |
| 283 | dataDir := t.TempDir() |
| 284 | if err := provision.Reconcile(ctx, w.boxA, boxReg, provision.ReconcileConfig{ |
| 285 | Region: fapiRegion, DataDir: dataDir, |
| 286 | }); err != nil { |
| 287 | t.Fatalf("(b) provision.Reconcile through the door: %v", err) |
| 288 | } |
| 289 | minted, err := boxReg.Get(fapiHost) |
| 290 | if err != nil || minted == nil { |
| 291 | t.Fatalf("(b) route %s = %+v, %v — the record never became a route", fapiHost, minted, err) |
| 292 | } |
| 293 | if minted.App != fapiApp || minted.Account != w.sqid || minted.Channel != fapiChannel || |
| 294 | !minted.Provisioned || minted.Seed != `["serve"]` { |
| 295 | t.Fatalf("(b) minted route = %+v, want the record's app/account/channel, Provisioned and the manifest seed", minted) |
| 296 | } |
| 297 | if rt, err := boxReg.Get("evil." + fapiDomain); err != nil || rt != nil { |
| 298 | t.Fatalf("(b) the other tenant's record minted a route on this box: %+v, %v", rt, err) |
| 299 | } |
| 300 | t.Log("(b) provision.Reconcile minted the fleet's own instance route and could not see the other tenant's") |
| 301 | |
| 302 | // --- (c) release adoption: pointer, artifact, staging -------------------- |
| 303 | // The whole adoption pass over the door: the signed channel pointer is |
| 304 | // fetched and verified, the manifest read, the artifact BODY streamed |
| 305 | // through the edge (F8 — no presigned redirect), and the version staged |
| 306 | // and stamped on the route. |
| 307 | artifact := make([]byte, 64<<10) |
| 308 | for i := range artifact { |
| 309 | artifact[i] = byte(i * 7) |
| 310 | } |
| 311 | if err := release.Ship(ctx, w.store, w.sqid, release.Manifest{ |
| 312 | App: fapiApp, Version: fapiVersion, Kind: "binary", |
| 313 | Artifacts: map[string]release.Artifact{fapiPlatform: {}}, |
| 314 | }, map[string][]byte{fapiPlatform: artifact}); err != nil { |
| 315 | t.Fatalf("(c) Ship: %v", err) |
| 316 | } |
| 317 | if err := release.Promote(ctx, w.store, w.priv, w.sqid, fapiApp, fapiChannel, fapiVersion); err != nil { |
| 318 | t.Fatalf("(c) Promote: %v", err) |
| 319 | } |
| 320 | optDir := t.TempDir() |
| 321 | var restarted []string |
| 322 | restart := func(h string) error { restarted = append(restarted, h); return nil } |
| 323 | if err := release.Pass(ctx, boxReg, w.boxA, w.pub, optDir, restart); err != nil { |
| 324 | t.Fatalf("(c) release.Pass through the door: %v", err) |
| 325 | } |
| 326 | if v, err := release.LiveVersion(optDir, fapiHost); v != fapiVersion { |
| 327 | t.Fatalf("(c) live version = %q (%v), want %q — adoption did not converge through the door", v, err, fapiVersion) |
| 328 | } |
| 329 | staged, err := os.ReadFile(release.VersionPath(optDir, w.sqid, fapiVersion, fapiApp)) |
| 330 | if err != nil || !slices.Equal(staged, artifact) { |
| 331 | t.Fatalf("(c) staged artifact = %d bytes (%v), want the shipped %d — a truncated body would pass a shorter check", len(staged), err, len(artifact)) |
| 332 | } |
| 333 | adopted, err := boxReg.Get(fapiHost) |
| 334 | if err != nil || adopted == nil || adopted.Version != fapiVersion { |
| 335 | t.Fatalf("(c) route version = %+v, %v; want %q stamped", adopted, err, fapiVersion) |
| 336 | } |
| 337 | t.Logf("(c) adoption converged: pointer verified, %d artifact bytes streamed and staged (restarted %v)", len(artifact), restarted) |
| 338 | |
| 339 | // --- (d) the heartbeat, under the Task-4 default prefix ------------------ |
| 340 | // A door-backed box gets CARLOS_STATUS_PREFIX defaulted to |
| 341 | // fleets/<fleet>/ (spec F5) precisely because the shared status/ root is |
| 342 | // outside the door's write ACL. The two lines statusLoop runs, with that |
| 343 | // rebase and the agent's label override: |
| 344 | prefixed := release.NewPrefixStore(w.boxA, "fleets/"+fapiFleet+"/") |
| 345 | routes, err := boxReg.List() |
| 346 | if err != nil { |
| 347 | t.Fatalf("(d) list box routes: %v", err) |
| 348 | } |
| 349 | box := status.CollectBox(routes, status.Probes{MachineID: "orchard-a", Region: "IE"}, time.Now()) |
| 350 | box.Label = fapiBoxA // statusTick's labelOverride: the fleet label, not a machine-id hash |
| 351 | if err := status.WriteBox(ctx, prefixed, box); err != nil { |
| 352 | t.Fatalf("(d) status.WriteBox through the door: %v", err) |
| 353 | } |
| 354 | heartbeatKey := "fleets/" + fapiFleet + "/status/boxes/" + fapiBoxA + ".json" |
| 355 | raw, _, err := w.store.Get(ctx, heartbeatKey) |
| 356 | if err != nil { |
| 357 | t.Fatalf("(d) heartbeat did not land at %s: %v", heartbeatKey, err) |
| 358 | } |
| 359 | var landed status.Box |
| 360 | if err := json.Unmarshal(raw, &landed); err != nil { |
| 361 | t.Fatalf("(d) heartbeat body: %v", err) |
| 362 | } |
| 363 | if landed.Label != fapiBoxA { |
| 364 | t.Fatalf("(d) heartbeat label = %q, want %q", landed.Label, fapiBoxA) |
| 365 | } |
| 366 | // The SECOND write is the one that exercises CAS through the door: |
| 367 | // WriteBox reads its own object's etag first (the read half of the fleet |
| 368 | // corner, which only exists so this replace can happen) and Puts with |
| 369 | // If-Match. A door that dropped etags would fail here, not above. |
| 370 | if err := status.WriteBox(ctx, prefixed, box); err != nil { |
| 371 | t.Fatalf("(d) heartbeat replace (read-etag-then-Put) through the door: %v", err) |
| 372 | } |
| 373 | // And a box may not heartbeat into a fleet that is not its own. |
| 374 | if _, err := w.boxA.Put(ctx, "fleets/other/status/boxes/box-a.json", []byte("{}"), ""); err == nil { |
| 375 | t.Fatal("(d) box-a wrote into another fleet's corner") |
| 376 | } |
| 377 | t.Logf("(d) heartbeat landed at %s and replaced itself under CAS", heartbeatKey) |
| 378 | |
| 379 | // --- (e) a fleet binding takeover, box → bucket → routing ---------------- |
| 380 | // The platform's route for the app, placed on the fleet and currently |
| 381 | // bound to box-b. |
| 382 | if err := w.reg.Upsert(registry.Route{ |
| 383 | Host: fapiHost, Kind: "instance", Status: "ready", |
| 384 | Region: registry.PlacementTarget(fapiFleet), FleetBox: fapiBoxB, |
| 385 | }); err != nil { |
| 386 | t.Fatalf("(e) upsert fleet-placed route: %v", err) |
| 387 | } |
| 388 | // box-b's agent-startup resident pass (takeover=false), through ITS door |
| 389 | // client — an observation, with no lease behind it. |
| 390 | if err := host.WriteFleetBinding(ctx, w.boxB, host.FleetBinding{ |
| 391 | Host: fapiHost, Fleet: fapiFleet, Box: fapiBoxB, BoundAt: time.Now().UnixMilli(), |
| 392 | }, false); err != nil { |
| 393 | t.Fatalf("(e) box-b resident binding: %v", err) |
| 394 | } |
| 395 | // box-a wakes the app and wins the lease (modeled in fleet_e2e_test.go; |
| 396 | // what matters here is the WRITE): a takeover claims the binding. |
| 397 | if err := host.WriteFleetBinding(ctx, w.boxA, host.FleetBinding{ |
| 398 | Host: fapiHost, Fleet: fapiFleet, Box: fapiBoxA, BoundAt: time.Now().UnixMilli(), |
| 399 | }, true); err != nil { |
| 400 | t.Fatalf("(e) box-a takeover binding through the door: %v", err) |
| 401 | } |
| 402 | // The platform reads it back from its own bucket handle — the object IS |
| 403 | // the message — and restamps routing. |
| 404 | if err := host.ReconcileFleetBindings(ctx, w.store, w.reg); err != nil { |
| 405 | t.Fatalf("(e) ReconcileFleetBindings: %v", err) |
| 406 | } |
| 407 | rt, err := w.reg.Get(fapiHost) |
| 408 | if err != nil || rt == nil || rt.FleetBox != fapiBoxA { |
| 409 | t.Fatalf("(e) Route.FleetBox = %+v, %v; want %q — the takeover never made it back into routing", rt, err, fapiBoxA) |
| 410 | } |
| 411 | // And the guard survives the round trip: box-b's next resident pass reads |
| 412 | // the existing binding through the door and declines to steal it back. |
| 413 | if err := host.WriteFleetBinding(ctx, w.boxB, host.FleetBinding{ |
| 414 | Host: fapiHost, Fleet: fapiFleet, Box: fapiBoxB, BoundAt: time.Now().UnixMilli(), |
| 415 | }, false); err != nil { |
| 416 | t.Fatalf("(e) box-b second resident pass: %v", err) |
| 417 | } |
| 418 | if err := host.ReconcileFleetBindings(ctx, w.store, w.reg); err != nil { |
| 419 | t.Fatalf("(e) ReconcileFleetBindings (after box-b's pass): %v", err) |
| 420 | } |
| 421 | if rt, err = w.reg.Get(fapiHost); err != nil || rt == nil || rt.FleetBox != fapiBoxA { |
| 422 | t.Fatalf("(e) Route.FleetBox = %+v, %v; want %q — a non-takeover write stole the binding through the door", rt, err, fapiBoxA) |
| 423 | } |
| 424 | // End to end, from outside: the edge's 2s poll picks the restamp up and |
| 425 | // the app is served by box-a's own channel. Record → route → release → |
| 426 | // binding → traffic, with the box holding no bucket credential anywhere |
| 427 | // in that chain. |
| 428 | waitBody(t, w.srvURL, fapiHost, fapiBoxA, e2eConvergeBound) |
| 429 | t.Log("(e) the takeover written through the door reached routing, and the bound box serves") |
| 430 | |
| 431 | // --- (f) the other tenant is invisible ---------------------------------- |
| 432 | // Listings first: the easiest place to leak a neighbour, and the one |
| 433 | // provision.Reconcile above depends on being right. |
| 434 | appKeys, err := w.boxA.List(ctx, "apps/") |
| 435 | if err != nil { |
| 436 | t.Fatalf("(f) list apps/: %v", err) |
| 437 | } |
| 438 | for _, k := range appKeys { |
| 439 | if !strings.HasPrefix(k, "apps/"+w.sqid+"/") { |
| 440 | t.Fatalf("(f) list apps/ returned %q — outside this fleet's account", k) |
| 441 | } |
| 442 | } |
| 443 | if len(appKeys) == 0 { |
| 444 | t.Fatal("(f) list apps/ came back empty — the scope map is refusing the fleet's OWN estate") |
| 445 | } |
| 446 | acctKeys, err := w.boxA.List(ctx, "accounts/") |
| 447 | if err != nil { |
| 448 | t.Fatalf("(f) list accounts/: %v", err) |
| 449 | } |
| 450 | if !slices.Equal(acctKeys, []string{"accounts/" + w.sqid + "/account.json"}) { |
| 451 | t.Fatalf("(f) list accounts/ = %v, want only this fleet's own account record", acctKeys) |
| 452 | } |
| 453 | acctDirs, err := w.boxA.ListDirs(ctx, "accounts/") |
| 454 | if err != nil { |
| 455 | t.Fatalf("(f) listdirs accounts/: %v", err) |
| 456 | } |
| 457 | if !slices.Equal(acctDirs, []string{"accounts/" + w.sqid + "/"}) { |
| 458 | t.Fatalf("(f) listdirs accounts/ = %v, want only this fleet's own directory (one child per account exists)", acctDirs) |
| 459 | } |
| 460 | fleetDirs, err := w.boxA.ListDirs(ctx, "fleets/") |
| 461 | if err != nil { |
| 462 | t.Fatalf("(f) listdirs fleets/: %v", err) |
| 463 | } |
| 464 | for _, d := range fleetDirs { |
| 465 | if !strings.HasPrefix(d, "fleets/"+fapiFleet+"/") { |
| 466 | t.Fatalf("(f) listdirs fleets/ returned %q — another fleet's corner", d) |
| 467 | } |
| 468 | } |
| 469 | // Then direct gets: every one of these keys exists in the bucket, and |
| 470 | // every one of them is os.ErrNotExist to this box. |
| 471 | for _, key := range []string{ |
| 472 | "accounts/" + w.other + "/account.json", |
| 473 | otherChannelKey, |
| 474 | otherLogKey, |
| 475 | "apps/" + w.other + "/" + fapiOtherApp + "/instances/evil." + fapiDomain + ".json", |
| 476 | "fleets/other/status/boxes/box-a.json", |
| 477 | } { |
| 478 | if _, _, err := w.store.Get(ctx, key); err != nil { |
| 479 | t.Fatalf("(f) fixture %s is not actually in the bucket: %v", key, err) |
| 480 | } |
| 481 | if _, _, err := w.boxA.Get(ctx, key); !errors.Is(err, os.ErrNotExist) { |
| 482 | t.Fatalf("(f) GET %s through the door = %v, want os.ErrNotExist (byte-identical 404)", key, err) |
| 483 | } |
| 484 | } |
| 485 | // Writes and deletes into the neighbour are refused too — including the |
| 486 | // delete of a log chunk, which is the one write verb the account-wide logs |
| 487 | // grant makes tempting. |
| 488 | if _, err := w.boxA.Put(ctx, otherChannelKey, []byte(`{"version":"evil"}`), ""); err == nil { |
| 489 | t.Fatal("(f) box-a promoted a release in another account") |
| 490 | } |
| 491 | if _, err := w.boxA.Put(ctx, otherLogKey, []byte("forged"), ""); err == nil { |
| 492 | t.Fatal("(f) box-a forged a log chunk in another account") |
| 493 | } |
| 494 | if err := w.boxA.Delete(ctx, otherLogKey); err == nil { |
| 495 | t.Fatal("(f) box-a deleted another account's log chunk") |
| 496 | } |
| 497 | if _, _, err := w.store.Get(ctx, otherChannelKey); err != nil { |
| 498 | t.Fatalf("(f) the refused writes changed the neighbour's object anyway: %v", err) |
| 499 | } |
| 500 | t.Log("(f) the second tenant is invisible to this fleet: listings, gets, writes and deletes all refuse") |
| 501 | |
| 502 | // --- (g) log chunks: ship and prune through the door --------------------- |
| 503 | // The Task-4 grant, exercised as logshipLoop uses it: chunks are written |
| 504 | // UNPREFIXED under the app's own subtree (applog.Prefix — where `carlos |
| 505 | // logs` reads them), listed back, and pruned by the same box. |
| 506 | hour := applog.HourOf(time.Now()) |
| 507 | chunk, err := applog.EncodeChunk([]applog.Record{{ |
| 508 | TS: time.Now(), Box: fapiBoxA, Host: fapiHost, Stream: "stdout", Msg: "listening", Seq: 1, |
| 509 | }}) |
| 510 | if err != nil { |
| 511 | t.Fatalf("(g) EncodeChunk: %v", err) |
| 512 | } |
| 513 | chunkKey, err := applog.ChunkKey(w.sqid, fapiApp, fapiBoxA, hour, 1, len(chunk)) |
| 514 | if err != nil { |
| 515 | t.Fatal(err) |
| 516 | } |
| 517 | if _, err := w.boxA.Put(ctx, chunkKey, chunk, ""); err != nil { |
| 518 | t.Fatalf("(g) ship a log chunk through the door: %v", err) |
| 519 | } |
| 520 | landedChunk, _, err := w.store.Get(ctx, chunkKey) |
| 521 | if err != nil { |
| 522 | t.Fatalf("(g) chunk did not land at %s: %v", chunkKey, err) |
| 523 | } |
| 524 | recs, err := applog.DecodeChunk(landedChunk) |
| 525 | if err != nil || len(recs) != 1 || recs[0].Msg != "listening" { |
| 526 | t.Fatalf("(g) chunk round trip = %+v, %v", recs, err) |
| 527 | } |
| 528 | boxPrefix, err := applog.BoxPrefix(w.sqid, fapiApp, fapiBoxA) |
| 529 | if err != nil { |
| 530 | t.Fatal(err) |
| 531 | } |
| 532 | listed, err := w.boxA.List(ctx, boxPrefix) |
| 533 | if err != nil || !slices.Contains(listed, chunkKey) { |
| 534 | t.Fatalf("(g) list own log subtree = %v, %v; want %s", listed, err, chunkKey) |
| 535 | } |
| 536 | if err := w.boxA.Delete(ctx, chunkKey); err != nil { |
| 537 | t.Fatalf("(g) prune own chunk through the door: %v", err) |
| 538 | } |
| 539 | if _, _, err := w.store.Get(ctx, chunkKey); !errors.Is(err, os.ErrNotExist) { |
| 540 | t.Fatalf("(g) chunk survived its prune: %v", err) |
| 541 | } |
| 542 | // The grant stays segment-exact: the same account, the same app, one |
| 543 | // segment over, is still refused. |
| 544 | for _, key := range []string{ |
| 545 | release.ChannelKey(w.sqid, fapiApp, fapiChannel), |
| 546 | provision.RecordKey(w.sqid, fapiApp, fapiHost), |
| 547 | "apps/" + w.sqid + "/" + fapiApp + "/config/production.json", |
| 548 | } { |
| 549 | if _, err := w.boxA.Put(ctx, key, []byte("{}"), ""); err == nil { |
| 550 | t.Fatalf("(g) the logs grant let the box write %s", key) |
| 551 | } |
| 552 | } |
| 553 | t.Log("(g) log chunks ship, list and prune through the door; neighbouring keys stay refused") |
| 554 | } |
| 555 | |
| 556 | // TestFleetBoxCatchallYieldsToAnotherTenantsClaim (#215 review): the store |
| 557 | // door withholds another account's claim BODY from a fleet box, but must |
| 558 | // still list its NAME. An instance catch-all derives no <label>.<parent> |
| 559 | // alias over a name somebody else has claimed, and ReconcileAliases decides |
| 560 | // "somebody else has claimed it" from the listing; a listing narrowed to the |
| 561 | // box's own account let the box shadow the other tenant's claim. |
| 562 | func TestFleetBoxCatchallYieldsToAnotherTenantsClaim(t *testing.T) { |
| 563 | w := newFleetDoorWorld(t) |
| 564 | ctx := context.Background() |
| 565 | const parent = "teams.example.com" |
| 566 | if _, _, err := domains.AttachClaim(ctx, w.store, domains.Claim{Host: parent, Account: w.sqid, App: "jelly"}); err != nil { |
| 567 | t.Fatal(err) |
| 568 | } |
| 569 | if _, err := domains.AttachInstanceCatchall(ctx, w.store, parent, w.sqid, "jelly", "", ""); err != nil { |
| 570 | t.Fatal(err) |
| 571 | } |
| 572 | // The other tenant's exact claim on the name the catch-all would derive. |
| 573 | if _, _, err := domains.AttachClaim(ctx, w.store, domains.Claim{ |
| 574 | Host: "beta." + parent, Account: w.other, App: "shop", Route: "shop." + w.other + ".oncarlos.com", |
| 575 | CreatedBy: "sam@keymail.dev", |
| 576 | }); err != nil { |
| 577 | t.Fatal(err) |
| 578 | } |
| 579 | instance := "beta." + w.sqid + ".oncarlos.com" |
| 580 | w.put(t, "apps/"+w.sqid+"/jelly/instances/"+instance+".json", `{}`) |
| 581 | // A second instance whose name nobody else claims: the control that |
| 582 | // shows the catch-all does derive through the door. |
| 583 | control := "acme." + w.sqid + ".oncarlos.com" |
| 584 | w.put(t, "apps/"+w.sqid+"/jelly/instances/"+control+".json", `{}`) |
| 585 | |
| 586 | // The box sees the other tenant's NAME but not its claim. |
| 587 | keys, err := w.boxA.List(ctx, domains.ClaimsPrefix) |
| 588 | if err != nil { |
| 589 | t.Fatal(err) |
| 590 | } |
| 591 | if !slices.Contains(keys, domains.ClaimKey("beta."+parent)) { |
| 592 | t.Fatalf("listing %v omits another tenant's claimed name", keys) |
| 593 | } |
| 594 | if _, _, err := w.boxA.Get(ctx, domains.ClaimKey("beta."+parent)); !errors.Is(err, os.ErrNotExist) { |
| 595 | t.Fatalf("another tenant's claim body: err = %v, want not-found", err) |
| 596 | } |
| 597 | |
| 598 | reg, err := registry.Open(filepath.Join(t.TempDir(), "box.db")) |
| 599 | if err != nil { |
| 600 | t.Fatal(err) |
| 601 | } |
| 602 | if err := reg.Upsert(registry.Route{Host: instance, Slug: "beta", App: "jelly", Account: w.sqid, Kind: "instance", Status: "ready"}); err != nil { |
| 603 | t.Fatal(err) |
| 604 | } |
| 605 | if err := reg.SetProvisioned(instance, true); err != nil { |
| 606 | t.Fatal(err) |
| 607 | } |
| 608 | if err := reg.Upsert(registry.Route{Host: control, Slug: "acme", App: "jelly", Account: w.sqid, Kind: "instance", Status: "ready"}); err != nil { |
| 609 | t.Fatal(err) |
| 610 | } |
| 611 | if err := reg.SetProvisioned(control, true); err != nil { |
| 612 | t.Fatal(err) |
| 613 | } |
| 614 | if err := domains.ReconcileAliases(ctx, w.boxA, reg, time.Now, nil, nil); err != nil { |
| 615 | t.Fatal(err) |
| 616 | } |
| 617 | aliases, err := reg.ListAliases() |
| 618 | if err != nil { |
| 619 | t.Fatal(err) |
| 620 | } |
| 621 | derived := false |
| 622 | for _, a := range aliases { |
| 623 | if a.Alias == "beta."+parent { |
| 624 | t.Fatalf("the box derived %+v over another tenant's claim", a) |
| 625 | } |
| 626 | derived = derived || (a.Alias == "acme."+parent && a.Host == control) |
| 627 | } |
| 628 | if !derived { |
| 629 | t.Fatalf("aliases = %+v, want acme.%s derived for the unclaimed name", aliases, parent) |
| 630 | } |
| 631 | } |
| 632 | |
| 633 | // Settings spec §4.1, ruling R17: a fleet box reads its own account's |
| 634 | // promotion records through the store door, so a build authorized by record |
| 635 | // alone (no pointer names it) stages on a fleet box, and a manifest that does |
| 636 | // not hash to the record's digest is refused. Another account's record and a |
| 637 | // listing of the records stay out of reach, and a box cannot write one. |
| 638 | func TestFleetBoxReadsItsOwnPromotionRecords(t *testing.T) { |
| 639 | ctx := context.Background() |
| 640 | w := newFleetDoorWorld(t) |
| 641 | const version = "v7" |
| 642 | for _, acct := range []string{w.sqid, w.other} { |
| 643 | if err := release.Ship(ctx, w.store, acct, release.Manifest{ |
| 644 | App: fapiApp, Version: version, Kind: "binary", |
| 645 | Artifacts: map[string]release.Artifact{fapiPlatform: {}}, |
| 646 | }, map[string][]byte{fapiPlatform: []byte("authorized bytes")}); err != nil { |
| 647 | t.Fatalf("Ship(%s): %v", acct, err) |
| 648 | } |
| 649 | _, raw, err := release.GetManifestRaw(ctx, w.store, acct, fapiApp, version) |
| 650 | if err != nil { |
| 651 | t.Fatal(err) |
| 652 | } |
| 653 | if err := release.RootSigner(w.priv).WritePromotionRecord(ctx, w.store, release.PromotionRecord{ |
| 654 | Account: acct, App: fapiApp, Version: version, ManifestDigest: release.ManifestDigest(raw), Authorized: true, |
| 655 | }); err != nil { |
| 656 | t.Fatalf("WritePromotionRecord(%s): %v", acct, err) |
| 657 | } |
| 658 | } |
| 659 | _, raw, err := release.GetManifestRaw(ctx, w.store, w.sqid, fapiApp, version) |
| 660 | if err != nil { |
| 661 | t.Fatal(err) |
| 662 | } |
| 663 | |
| 664 | rec, err := release.ReadPromotionRecord(ctx, w.boxA, w.pub, w.sqid, fapiApp, version) |
| 665 | if err != nil || rec.ManifestDigest != release.ManifestDigest(raw) { |
| 666 | t.Fatalf("ReadPromotionRecord through the door = %+v, %v; want the record signing %s", rec, err, release.ManifestDigest(raw)) |
| 667 | } |
| 668 | if m, err := release.AuthorizedManifest(ctx, w.boxA, w.pub, w.sqid, fapiApp, version); err != nil || m.Version != version { |
| 669 | t.Fatalf("AuthorizedManifest through the door = %+v, %v; want the record-only authorized %s", m, err, version) |
| 670 | } |
| 671 | |
| 672 | if _, err := release.ReadPromotionRecord(ctx, w.boxA, w.pub, w.other, fapiApp, version); err == nil { |
| 673 | t.Fatal("a fleet box read another account's promotion record through the door") |
| 674 | } |
| 675 | keys, _ := w.boxA.List(ctx, "releases/"+w.sqid+"/") |
| 676 | for _, k := range keys { |
| 677 | if strings.HasPrefix(k, "releases/") { |
| 678 | t.Fatalf("a fleet box listed promotion records through the door: %v", keys) |
| 679 | } |
| 680 | } |
| 681 | if _, err := w.boxA.Put(ctx, release.PromotionRecordKey(w.sqid, fapiApp, "v8"), []byte("{}"), ""); err == nil { |
| 682 | t.Fatal("a fleet box wrote a promotion record through the door") |
| 683 | } |
| 684 | |
| 685 | _, etag, err := w.store.Get(ctx, release.ManifestKey(w.sqid, fapiApp, version)) |
| 686 | if err != nil { |
| 687 | t.Fatal(err) |
| 688 | } |
| 689 | if _, err := w.store.Put(ctx, release.ManifestKey(w.sqid, fapiApp, version), append(raw, '\n'), etag); err != nil { |
| 690 | t.Fatal(err) |
| 691 | } |
| 692 | if _, err := release.AuthorizedManifest(ctx, w.boxA, w.pub, w.sqid, fapiApp, version); !errors.Is(err, release.ErrManifestDigest) { |
| 693 | t.Fatalf("AuthorizedManifest over a replaced manifest = %v; want ErrManifestDigest", err) |
| 694 | } |
| 695 | } |
| 696 | |
amadan