| 1 | package provision |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "fmt" |
| 6 | "os" |
| 7 | "path/filepath" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/registry" |
| 13 | "github.com/carlosframework/platform/internal/release" |
| 14 | ) |
| 15 | |
| 16 | // --- fixtures ----------------------------------------------------------------- |
| 17 | |
| 18 | // failStore is the read-failure seam these tests need and no other store |
| 19 | // shape provides: a Store whose Get (and List) refuse a chosen prefix, so a |
| 20 | // pass can be put in exactly the state the fail-safe exists for — a record |
| 21 | // that IS there and could not be read — rather than the state that looks the |
| 22 | // same from the outside and means the opposite (a record the bucket retired). |
| 23 | type failStore struct { |
| 24 | release.Store |
| 25 | fail []string |
| 26 | } |
| 27 | |
| 28 | func (s *failStore) failReadsFor(prefix string) { s.fail = append(s.fail, prefix) } |
| 29 | |
| 30 | func (s *failStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 31 | for _, p := range s.fail { |
| 32 | if strings.HasPrefix(key, p) { |
| 33 | return nil, "", fmt.Errorf("failStore: reads disabled under %s", p) |
| 34 | } |
| 35 | } |
| 36 | return s.Store.Get(ctx, key) |
| 37 | } |
| 38 | |
| 39 | func (s *failStore) List(ctx context.Context, prefix string) ([]string, error) { |
| 40 | for _, p := range s.fail { |
| 41 | if strings.HasPrefix(prefix, p) { |
| 42 | return nil, fmt.Errorf("failStore: reads disabled under %s", p) |
| 43 | } |
| 44 | } |
| 45 | return s.Store.List(ctx, prefix) |
| 46 | } |
| 47 | |
| 48 | type sidecarFixture struct { |
| 49 | store *failStore |
| 50 | reg *registry.Registry |
| 51 | dir string |
| 52 | sqid string |
| 53 | } |
| 54 | |
| 55 | // sidecarSetup writes a valid account manifest and hands back a fixture whose |
| 56 | // data dir and registry are empty — the sidecar family's equivalent of |
| 57 | // setup() above, minus the instances manifest (a sidecar app opts in through |
| 58 | // its OWN manifest, sidecars.json). |
| 59 | func sidecarSetup(t *testing.T) *sidecarFixture { |
| 60 | t.Helper() |
| 61 | s := &failStore{Store: dirStore(t)} |
| 62 | sqid := writeAccountManifest(t, s, acctID, "Eleven") |
| 63 | return &sidecarFixture{store: s, reg: regWith(t), dir: t.TempDir(), sqid: sqid} |
| 64 | } |
| 65 | |
| 66 | func (f *sidecarFixture) host(name, app string) string { return SidecarHost(name, app, f.sqid) } |
| 67 | |
| 68 | func (f *sidecarFixture) putManifest(t *testing.T, app string, m SidecarManifest) { |
| 69 | t.Helper() |
| 70 | if err := PutSidecarManifest(ctx, f.store, f.sqid, app, m); err != nil { |
| 71 | t.Fatal(err) |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | func (f *sidecarFixture) putRecord(t *testing.T, rec SidecarRecord) { |
| 76 | t.Helper() |
| 77 | if rec.Account == "" { |
| 78 | rec.Account = f.sqid |
| 79 | } |
| 80 | if rec.Host == "" { |
| 81 | rec.Host = SidecarHost(rec.Name, rec.App, rec.Account) |
| 82 | } |
| 83 | if err := CreateSidecarRecord(ctx, f.store, rec); err != nil { |
| 84 | t.Fatal(err) |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | func (f *sidecarFixture) mustUpsert(t *testing.T, rt registry.Route) { |
| 89 | t.Helper() |
| 90 | if err := f.reg.Upsert(rt); err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | func (f *sidecarFixture) reconcile(t *testing.T, cfg ReconcileConfig) { |
| 96 | t.Helper() |
| 97 | cfg.DataDir = f.dir |
| 98 | if err := ReconcileSidecars(ctx, f.store, f.reg, cfg); err != nil { |
| 99 | t.Fatalf("ReconcileSidecars: %v", err) |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | // noRoute asserts a host has no row. registry.Get answers a missing host with |
| 104 | // (nil, nil), so a bare `err == nil` check would pass whether the row is |
| 105 | // there or not. |
| 106 | func noRoute(t *testing.T, reg *registry.Registry, host, msg string) { |
| 107 | t.Helper() |
| 108 | rt, err := reg.Get(host) |
| 109 | if err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | if rt != nil { |
| 113 | t.Fatal(msg) |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | func hasRoute(t *testing.T, reg *registry.Registry, host, msg string) *registry.Route { |
| 118 | t.Helper() |
| 119 | rt, err := reg.Get(host) |
| 120 | if err != nil { |
| 121 | t.Fatal(err) |
| 122 | } |
| 123 | if rt == nil { |
| 124 | t.Fatal(msg) |
| 125 | } |
| 126 | return rt |
| 127 | } |
| 128 | |
| 129 | // --- the removal-authority collision ------------------------------------------- |
| 130 | |
| 131 | // TestInstanceReconcileNeverRemovesSidecarRows is the guard on the OTHER |
| 132 | // pass. A sidecar row is Provisioned (that flag is what grants a pass removal |
| 133 | // authority) and can never appear in the instance pass's record set, so |
| 134 | // without a Kind guard the instance pass deletes every sidecar row on every |
| 135 | // tick — minted by one pass, destroyed by the other, forever, with no error |
| 136 | // anywhere and nothing in the logs that reads as a failure. |
| 137 | func TestInstanceReconcileNeverRemovesSidecarRows(t *testing.T) { |
| 138 | f := sidecarSetup(t) |
| 139 | host := f.host("bot", "eleven") |
| 140 | f.mustUpsert(t, registry.Route{Host: host, Kind: "sidecar", Slug: "bot", |
| 141 | App: "eleven", Account: f.sqid, Provisioned: true, Status: "ready"}) |
| 142 | |
| 143 | // No instance records at all: the pass's removal phase runs in full. |
| 144 | if err := Reconcile(ctx, f.store, f.reg, ReconcileConfig{DataDir: f.dir}); err != nil { |
| 145 | t.Fatalf("Reconcile: %v", err) |
| 146 | } |
| 147 | hasRoute(t, f.reg, host, "the instance pass deleted a sidecar row — each pass's removal authority is scoped to its OWN kind, or the two passes destroy each other's work forever") |
| 148 | } |
| 149 | |
| 150 | // TestSidecarReconcileRemovesOnlyItsOwnKind is the same guard from the other |
| 151 | // side: this pass retires its own kind and must never reach an instance row, |
| 152 | // which can never appear in ITS record set either. |
| 153 | func TestSidecarReconcileRemovesOnlyItsOwnKind(t *testing.T) { |
| 154 | f := sidecarSetup(t) |
| 155 | gone := f.host("gone", "eleven") |
| 156 | f.mustUpsert(t, registry.Route{Host: gone, Kind: "sidecar", Slug: "gone", |
| 157 | App: "eleven", Account: f.sqid, Provisioned: true}) |
| 158 | f.mustUpsert(t, registry.Route{Host: "app.example.com", Kind: "instance", |
| 159 | App: "eleven", Account: f.sqid, Provisioned: true}) |
| 160 | // A sidecar-kind row nothing provisioned: Provisioned is the flag that |
| 161 | // GRANTS removal authority, so a row without it is not this pass's to |
| 162 | // retire however well it matches on every other column. |
| 163 | hand := f.host("hand", "eleven") |
| 164 | f.mustUpsert(t, registry.Route{Host: hand, Kind: "sidecar", Slug: "hand", |
| 165 | App: "eleven", Account: f.sqid}) |
| 166 | |
| 167 | f.reconcile(t, ReconcileConfig{}) |
| 168 | |
| 169 | noRoute(t, f.reg, gone, "a provisioned sidecar with no record must be removed") |
| 170 | hasRoute(t, f.reg, "app.example.com", "the sidecar pass must never touch an instance row") |
| 171 | hasRoute(t, f.reg, hand, "a sidecar row that was never provisioned is not this pass's to remove") |
| 172 | } |
| 173 | |
| 174 | // --- mint / converge / remove -------------------------------------------------- |
| 175 | |
| 176 | func TestSidecarReconcileMints(t *testing.T) { |
| 177 | f := sidecarSetup(t) |
| 178 | f.putManifest(t, "eleven", SidecarManifest{ |
| 179 | Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}, Channel: "stable"}) |
| 180 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"}) |
| 181 | |
| 182 | f.reconcile(t, ReconcileConfig{}) |
| 183 | |
| 184 | rt := hasRoute(t, f.reg, f.host("bot", "eleven"), "no sidecar route minted") |
| 185 | if rt.Kind != "sidecar" { |
| 186 | t.Fatalf("Kind = %q", rt.Kind) |
| 187 | } |
| 188 | if rt.Socket != "" || rt.Addr != "" { |
| 189 | t.Fatalf("a sidecar row must carry NO socket and NO addr, got socket=%q addr=%q", rt.Socket, rt.Addr) |
| 190 | } |
| 191 | if rt.Slug != "bot" || rt.App != "eleven" || rt.Account != f.sqid || rt.Channel != "stable" { |
| 192 | t.Fatalf("route identity = %+v", rt) |
| 193 | } |
| 194 | if rt.DBPath != DefaultDBPath(f.dir, rt.Host) || !rt.Provisioned || rt.Args != `["sidecar","run"]` { |
| 195 | t.Fatalf("route = %+v", rt) |
| 196 | } |
| 197 | if !rt.Hibernate { |
| 198 | t.Fatal("Hibernate defaults true (HibernateOrDefault), and the row is what the activator reads") |
| 199 | } |
| 200 | if rt.IdleSeconds != 300 || rt.MaxRunSeconds != 3600 { |
| 201 | t.Fatalf("ceilings = %d/%d, want the manifest defaults 300/3600 materialized onto the row — the sweeper must never need a bucket read", |
| 202 | rt.IdleSeconds, rt.MaxRunSeconds) |
| 203 | } |
| 204 | } |
| 205 | |
| 206 | func TestSidecarReconcileConvergesArgsFromTheManifest(t *testing.T) { |
| 207 | f := sidecarSetup(t) |
| 208 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}}) |
| 209 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"}) |
| 210 | f.reconcile(t, ReconcileConfig{}) |
| 211 | |
| 212 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"worker", "run"}}) |
| 213 | f.reconcile(t, ReconcileConfig{}) |
| 214 | |
| 215 | rt := hasRoute(t, f.reg, f.host("bot", "eleven"), "route gone") |
| 216 | if rt.Args != `["worker","run"]` { |
| 217 | t.Fatalf("Args = %q — truth lives in the bucket and is re-stamped every pass, exactly as seed is", rt.Args) |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | func TestSidecarReconcileConvergesTheCeilings(t *testing.T) { |
| 222 | f := sidecarSetup(t) |
| 223 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", |
| 224 | Args: []string{"sidecar", "run"}, IdleSeconds: 300, MaxRunSeconds: 3600}) |
| 225 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"}) |
| 226 | f.reconcile(t, ReconcileConfig{}) |
| 227 | |
| 228 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", |
| 229 | Args: []string{"sidecar", "run"}, IdleSeconds: 60, MaxRunSeconds: 120}) |
| 230 | f.reconcile(t, ReconcileConfig{}) |
| 231 | |
| 232 | rt := hasRoute(t, f.reg, f.host("bot", "eleven"), "route gone") |
| 233 | if rt.IdleSeconds != 60 || rt.MaxRunSeconds != 120 { |
| 234 | t.Fatalf("ceilings = %d/%d, want 60/120 — the sweeper reads the row, so the row must follow the manifest", rt.IdleSeconds, rt.MaxRunSeconds) |
| 235 | } |
| 236 | } |
| 237 | |
| 238 | // TestSidecarReconcileConvergesChannelEnvironmentAndHibernate pins the rest |
| 239 | // of the owned columns: the record owns channel and environment (the #130 |
| 240 | // discipline), and the manifest owns hibernation. |
| 241 | func TestSidecarReconcileConvergesChannelEnvironmentAndHibernate(t *testing.T) { |
| 242 | f := sidecarSetup(t) |
| 243 | off := false |
| 244 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", |
| 245 | Args: []string{"sidecar", "run"}, Hibernate: &off}) |
| 246 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", |
| 247 | Channel: "canary/eleven", Environment: "relay"}) |
| 248 | f.reconcile(t, ReconcileConfig{}) |
| 249 | |
| 250 | rt := hasRoute(t, f.reg, f.host("bot", "eleven"), "route gone") |
| 251 | if rt.Channel != "canary/eleven" || rt.Environment != "relay" || rt.Hibernate { |
| 252 | t.Fatalf("minted route = channel %q env %q hibernate %v", rt.Channel, rt.Environment, rt.Hibernate) |
| 253 | } |
| 254 | |
| 255 | // Drift the row the way `carlos route` would, then converge it back. |
| 256 | if err := f.reg.SetChannel(rt.Host, "stable"); err != nil { |
| 257 | t.Fatal(err) |
| 258 | } |
| 259 | if err := f.reg.SetEnvironment(rt.Host, "other"); err != nil { |
| 260 | t.Fatal(err) |
| 261 | } |
| 262 | if err := f.reg.SetHibernate(rt.Host, true); err != nil { |
| 263 | t.Fatal(err) |
| 264 | } |
| 265 | f.reconcile(t, ReconcileConfig{}) |
| 266 | |
| 267 | rt = hasRoute(t, f.reg, f.host("bot", "eleven"), "route gone") |
| 268 | if rt.Channel != "canary/eleven" || rt.Environment != "relay" || rt.Hibernate { |
| 269 | t.Fatalf("after converge = channel %q env %q hibernate %v — the record and manifest own these columns", |
| 270 | rt.Channel, rt.Environment, rt.Hibernate) |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | // --- the fail-safe -------------------------------------------------------------- |
| 275 | |
| 276 | // TestSidecarReconcileSkipsRemovalWhenAReadFailed pins the fail-safe: a |
| 277 | // transient read failure must never mass-retire a tenant's fleet. Minting |
| 278 | // still proceeds, because minting is additive. |
| 279 | func TestSidecarReconcileSkipsRemovalWhenAReadFailed(t *testing.T) { |
| 280 | f := sidecarSetup(t) |
| 281 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}}) |
| 282 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"}) |
| 283 | keep := f.host("keep", "eleven") |
| 284 | f.mustUpsert(t, registry.Route{Host: keep, Kind: "sidecar", Slug: "keep", |
| 285 | App: "eleven", Account: f.sqid, Provisioned: true}) |
| 286 | |
| 287 | // The record IS there; this pass simply cannot read it. |
| 288 | f.store.failReadsFor(SidecarsPrefix(f.sqid, "eleven")) |
| 289 | f.reconcile(t, ReconcileConfig{}) |
| 290 | |
| 291 | hasRoute(t, f.reg, keep, "a read failure skipped nothing — one bad list must not retire a fleet") |
| 292 | } |
| 293 | |
| 294 | // TestSidecarReconcileProtectsAMalformedRecord is the narrower guard: only |
| 295 | // the host whose body could not be trusted is held back, and it is held back |
| 296 | // rather than read as "the bucket retired it" — a non-additive record |
| 297 | // encoding change reaching an older binary would otherwise delete every |
| 298 | // sidecar in scope in one tick. |
| 299 | func TestSidecarReconcileProtectsAMalformedRecord(t *testing.T) { |
| 300 | f := sidecarSetup(t) |
| 301 | host := f.host("bot", "eleven") |
| 302 | f.mustUpsert(t, registry.Route{Host: host, Kind: "sidecar", Slug: "bot", |
| 303 | App: "eleven", Account: f.sqid, Provisioned: true}) |
| 304 | if _, err := f.store.Put(ctx, SidecarRecordKey(f.sqid, "eleven", "bot"), []byte("{not json"), ""); err != nil { |
| 305 | t.Fatal(err) |
| 306 | } |
| 307 | |
| 308 | f.reconcile(t, ReconcileConfig{}) |
| 309 | |
| 310 | hasRoute(t, f.reg, host, "a record that cannot be PARSED is not a record that was retired") |
| 311 | } |
| 312 | |
| 313 | // TestSidecarReconcileProtectsAChannellessRecord: a route with no channel |
| 314 | // resolves no release, so it has no build to exec — the worker boot pass |
| 315 | // would try to run it against a symlink that does not exist. The console |
| 316 | // stamps a channel at create, so a record without one is malformed, and |
| 317 | // malformed is exactly what protect is for: not minted, and not read as "the |
| 318 | // bucket retired it" either. |
| 319 | func TestSidecarReconcileProtectsAChannellessRecord(t *testing.T) { |
| 320 | f := sidecarSetup(t) |
| 321 | host := f.host("bot", "eleven") |
| 322 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", |
| 323 | Args: []string{"sidecar", "run"}, Channel: "stable"}) |
| 324 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven"}) // no channel, no row yet |
| 325 | // A channel-less record whose row already exists: protected, not retired. |
| 326 | held := f.host("held", "eleven") |
| 327 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "held", App: "eleven"}) |
| 328 | f.mustUpsert(t, registry.Route{Host: held, Kind: "sidecar", Slug: "held", |
| 329 | App: "eleven", Account: f.sqid, Channel: "stable", Provisioned: true}) |
| 330 | // A row with no record at all, to prove the removal phase still ran. |
| 331 | old := f.host("old", "eleven") |
| 332 | f.mustUpsert(t, registry.Route{Host: old, Kind: "sidecar", Slug: "old", |
| 333 | App: "eleven", Account: f.sqid, Provisioned: true}) |
| 334 | |
| 335 | f.reconcile(t, ReconcileConfig{}) |
| 336 | |
| 337 | noRoute(t, f.reg, host, "a channel-less record must not mint a row that can never resolve a release — the manifest's channel is the console door's default at CREATE time, not a runtime rescue") |
| 338 | hasRoute(t, f.reg, held, "a record that cannot be USED is not a record that was retired") |
| 339 | noRoute(t, f.reg, old, "protect is narrow: an unusable record holds back ITS OWN host only, and the pass's removal phase still runs") |
| 340 | } |
| 341 | |
| 342 | // --- decline-but-keep ------------------------------------------------------------- |
| 343 | |
| 344 | // TestSidecarReconcileKeepsRowsWhenTheManifestIsAbsent pins the invariant that |
| 345 | // stands between "an app's sidecars.json was deleted" and "every sidecar row |
| 346 | // of that app is retired". An absent manifest is a confirmed answer, not a |
| 347 | // blip, so it does NOT arm the pass-wide fail-safe — which is exactly why the |
| 348 | // record must stay in the record set on its own account. Declining to mint is |
| 349 | // the whole of the response. |
| 350 | // |
| 351 | // The hazard is one line away: manifestFor is consulted in the record scan |
| 352 | // (placement needs it), so a natural-looking `if !mKnown { continue }` there |
| 353 | // would convert a missing manifest into a fleet evacuation. |
| 354 | func TestSidecarReconcileKeepsRowsWhenTheManifestIsAbsent(t *testing.T) { |
| 355 | f := sidecarSetup(t) |
| 356 | host := f.host("bot", "eleven") |
| 357 | // No sidecars.json for this app at all — the feature is not enabled. |
| 358 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"}) |
| 359 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "new", App: "eleven", Channel: "stable"}) |
| 360 | f.mustUpsert(t, registry.Route{Host: host, Kind: "sidecar", Slug: "bot", App: "eleven", |
| 361 | Account: f.sqid, Channel: "stable", Provisioned: true, Args: `["sidecar","run"]`}) |
| 362 | |
| 363 | f.reconcile(t, ReconcileConfig{}) |
| 364 | |
| 365 | rt := hasRoute(t, f.reg, host, "an absent sidecars manifest must not RETIRE a running sidecar — it is the app's opt-in door, not a statement that this row is gone") |
| 366 | if rt.Args != `["sidecar","run"]` { |
| 367 | t.Fatalf("Args = %q — with no manifest there is nothing authoritative to converge from, so the row must be left exactly as it was", rt.Args) |
| 368 | } |
| 369 | noRoute(t, f.reg, f.host("new", "eleven"), "with no manifest there is no argv to run, so nothing may be minted either") |
| 370 | } |
| 371 | |
| 372 | // TestSidecarReconcileKeepsRowsOfATrashedApp: a trashed app mints nothing |
| 373 | // here, and its rows are not this pass's to retire either. domains.Reconcile |
| 374 | // drops every route of a trashed app on its own authority (with its own |
| 375 | // production-flag guard) — this pass merely declines to create, or the two |
| 376 | // loops would fight forever, one re-raising what the other removes. |
| 377 | func TestSidecarReconcileKeepsRowsOfATrashedApp(t *testing.T) { |
| 378 | f := sidecarSetup(t) |
| 379 | host := f.host("bot", "eleven") |
| 380 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}}) |
| 381 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"}) |
| 382 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "new", App: "eleven", Channel: "stable"}) |
| 383 | f.mustUpsert(t, registry.Route{Host: host, Kind: "sidecar", Slug: "bot", App: "eleven", |
| 384 | Account: f.sqid, Channel: "stable", Provisioned: true}) |
| 385 | if err := release.TrashApp(ctx, f.store, f.sqid, "eleven", "someone@example.com", time.Now()); err != nil { |
| 386 | t.Fatal(err) |
| 387 | } |
| 388 | |
| 389 | f.reconcile(t, ReconcileConfig{}) |
| 390 | |
| 391 | hasRoute(t, f.reg, host, "dropping a trashed app's routes is domains.Reconcile's call, with its production-flag guard — this pass only declines to create") |
| 392 | noRoute(t, f.reg, f.host("new", "eleven"), "a trashed app provisions nothing") |
| 393 | } |
| 394 | |
| 395 | // --- placement ------------------------------------------------------------------- |
| 396 | |
| 397 | // TestSidecarReconcilePlacementGate pins §8's second half: a no-inbound |
| 398 | // workload lands only on a box that declares it accepts no inbound work. Both |
| 399 | // spellings are honoured — the record's own placement and the app manifest's. |
| 400 | func TestSidecarReconcilePlacementGate(t *testing.T) { |
| 401 | f := sidecarSetup(t) |
| 402 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}}) |
| 403 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable", Placement: PlacementNoInbound}) |
| 404 | // A second app whose MANIFEST asks for the placement, record silent. |
| 405 | f.putManifest(t, "relay", SidecarManifest{Schema: 1, App: "relay", |
| 406 | Args: []string{"relay"}, Placement: PlacementNoInbound}) |
| 407 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "hub", App: "relay", Channel: "stable"}) |
| 408 | |
| 409 | f.reconcile(t, ReconcileConfig{}) // an ordinary box: accepts inbound work |
| 410 | noRoute(t, f.reg, f.host("bot", "eleven"), "a no-inbound workload must not land on a box that accepts inbound work") |
| 411 | noRoute(t, f.reg, f.host("hub", "relay"), "a manifest-stated no-inbound placement must gate exactly as a record-stated one does") |
| 412 | |
| 413 | f.reconcile(t, ReconcileConfig{Inbound: InboundNone}) |
| 414 | hasRoute(t, f.reg, f.host("bot", "eleven"), "it must land on a box that declares CARLOS_INBOUND=none") |
| 415 | hasRoute(t, f.reg, f.host("hub", "relay"), "it must land on a box that declares CARLOS_INBOUND=none") |
| 416 | } |
| 417 | |
| 418 | // TestSidecarReconcilePlacementIsMostRestrictive: a record may NARROW the |
| 419 | // app's placement but never widen it. "any" on a record under a no-inbound |
| 420 | // manifest is the widening direction — the console door refuses to write it |
| 421 | // at all, and this pass must refuse to honour one that reached the bucket by |
| 422 | // any other route, or app-wide policy is only a suggestion. |
| 423 | func TestSidecarReconcilePlacementIsMostRestrictive(t *testing.T) { |
| 424 | f := sidecarSetup(t) |
| 425 | // App-wide: everything this app runs sits somewhere with no inbound path. |
| 426 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", |
| 427 | Args: []string{"sidecar", "run"}, Placement: PlacementNoInbound}) |
| 428 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable", Placement: PlacementAny}) |
| 429 | |
| 430 | f.reconcile(t, ReconcileConfig{}) // an ordinary box: accepts inbound work |
| 431 | noRoute(t, f.reg, f.host("bot", "eleven"), |
| 432 | `a record's "any" must not override the app's no-inbound placement — that is the widening direction the console door already refuses`) |
| 433 | |
| 434 | f.reconcile(t, ReconcileConfig{Inbound: InboundNone}) |
| 435 | hasRoute(t, f.reg, f.host("bot", "eleven"), "it still lands where the app's own policy says it may") |
| 436 | } |
| 437 | |
| 438 | // The narrowing direction is untouched: one bot that must run with no inbound |
| 439 | // path, under an app that states nothing, is honoured. |
| 440 | func TestSidecarReconcileRecordMayNarrowAnUnplacedApp(t *testing.T) { |
| 441 | f := sidecarSetup(t) |
| 442 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}}) |
| 443 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable", Placement: PlacementNoInbound}) |
| 444 | |
| 445 | f.reconcile(t, ReconcileConfig{}) |
| 446 | noRoute(t, f.reg, f.host("bot", "eleven"), "a record asking for no-inbound must not land on an inbound box") |
| 447 | f.reconcile(t, ReconcileConfig{Inbound: InboundNone}) |
| 448 | hasRoute(t, f.reg, f.host("bot", "eleven"), "a record may narrow an app that states no placement of its own") |
| 449 | } |
| 450 | |
| 451 | // TestSidecarReconcilePlacementChangeEvacuatesTheRow: placement is a SCOPE |
| 452 | // axis, beside region and pool, so flipping a manifest to no-inbound takes the |
| 453 | // row off a box that accepts inbound work — the same way repointing a |
| 454 | // record's region moves an instance. Decline-but-keep would break the only |
| 455 | // guarantee the axis exists to give: a member who asks that a bot only ever |
| 456 | // run where there is no inbound path would find it still running on an |
| 457 | // inbound box. |
| 458 | func TestSidecarReconcilePlacementChangeEvacuatesTheRow(t *testing.T) { |
| 459 | f := sidecarSetup(t) |
| 460 | host := f.host("bot", "eleven") |
| 461 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}}) |
| 462 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"}) |
| 463 | f.reconcile(t, ReconcileConfig{}) |
| 464 | hasRoute(t, f.reg, host, "an unplaced record mints here") |
| 465 | |
| 466 | // The app's owner now requires no inbound path anywhere it runs. |
| 467 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", |
| 468 | Args: []string{"sidecar", "run"}, Placement: PlacementNoInbound}) |
| 469 | |
| 470 | f.reconcile(t, ReconcileConfig{}) // this box accepts inbound work |
| 471 | noRoute(t, f.reg, host, "flipping placement to no-inbound must EVACUATE the row from an inbound box, not leave the bot running there") |
| 472 | |
| 473 | f.reconcile(t, ReconcileConfig{Inbound: InboundNone}) // and land where it belongs |
| 474 | hasRoute(t, f.reg, host, "the same record must mint on a box that declares CARLOS_INBOUND=none") |
| 475 | } |
| 476 | |
| 477 | // TestSidecarReconcilePlacementEvacuationObeysTheFailSafe: evacuation is a |
| 478 | // removal, so it is governed by the pass-wide fail-safe like every other |
| 479 | // removal. A read that failed anywhere this pass means this box cannot tell a |
| 480 | // placement change from a blip, and a transient blip must never evacuate a |
| 481 | // tenant's fleet. |
| 482 | func TestSidecarReconcilePlacementEvacuationObeysTheFailSafe(t *testing.T) { |
| 483 | f := sidecarSetup(t) |
| 484 | host := f.host("bot", "eleven") |
| 485 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", |
| 486 | Args: []string{"sidecar", "run"}, Placement: PlacementNoInbound}) |
| 487 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"}) |
| 488 | f.mustUpsert(t, registry.Route{Host: host, Kind: "sidecar", Slug: "bot", |
| 489 | App: "eleven", Account: f.sqid, Provisioned: true}) |
| 490 | // An UNRELATED app's records cannot be read this pass. |
| 491 | f.putManifest(t, "relay", SidecarManifest{Schema: 1, App: "relay", Args: []string{"relay"}}) |
| 492 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "hub", App: "relay", Channel: "stable"}) |
| 493 | f.store.failReadsFor(SidecarsPrefix(f.sqid, "relay")) |
| 494 | |
| 495 | f.reconcile(t, ReconcileConfig{}) // placement does not match, but a read failed |
| 496 | |
| 497 | hasRoute(t, f.reg, host, "a pass that could not read everything must evacuate nothing — the fail-safe governs placement removals too") |
| 498 | } |
| 499 | |
| 500 | // --- the co-spawn conflict --------------------------------------------------------- |
| 501 | |
| 502 | // TestSidecarReconcileRefusesHostWithAStagedCoSpawnFile: the file-staged |
| 503 | // co-spawn is legacy but LIVE (Eleven runs on it today). One host, one |
| 504 | // mechanism. |
| 505 | func TestSidecarReconcileRefusesHostWithAStagedCoSpawnFile(t *testing.T) { |
| 506 | f := sidecarSetup(t) |
| 507 | host := f.host("bot", "eleven") |
| 508 | if err := os.WriteFile(filepath.Join(f.dir, host+"-sidecar.env"), []byte("X=1\n"), 0o600); err != nil { |
| 509 | t.Fatal(err) |
| 510 | } |
| 511 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}}) |
| 512 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"}) |
| 513 | |
| 514 | f.reconcile(t, ReconcileConfig{}) |
| 515 | |
| 516 | noRoute(t, f.reg, host, "a host with a staged legacy co-spawn file must not also get a native sidecar row — one host, one mechanism") |
| 517 | } |
| 518 | |
| 519 | // --- scope --------------------------------------------------------------------------- |
| 520 | |
| 521 | // TestSidecarReconcileSkipsAnotherBoxesScope: region and pool scope the |
| 522 | // sidecar pass exactly as they scope the instance one — a record for another |
| 523 | // agent's jurisdiction is neither minted nor (for the row) removed here. |
| 524 | func TestSidecarReconcileSkipsAnotherBoxesScope(t *testing.T) { |
| 525 | f := sidecarSetup(t) |
| 526 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}}) |
| 527 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable", Region: "ie", Pool: "eleven"}) |
| 528 | // A foreign box's own provisioned sidecar row, visible in this registry. |
| 529 | foreign := f.host("far", "eleven") |
| 530 | f.mustUpsert(t, registry.Route{Host: foreign, Kind: "sidecar", Slug: "far", App: "eleven", |
| 531 | Account: f.sqid, Region: "de", Pool: "other", Provisioned: true}) |
| 532 | |
| 533 | f.reconcile(t, ReconcileConfig{}) // this box: no region, no pools |
| 534 | |
| 535 | noRoute(t, f.reg, f.host("bot", "eleven"), "a record for another region/pool is not this pass's business") |
| 536 | hasRoute(t, f.reg, foreign, "a foreign box's provisioned route is not this pass's to retire") |
| 537 | |
| 538 | f.reconcile(t, ReconcileConfig{Region: "ie", Pools: []string{"eleven"}}) |
| 539 | hasRoute(t, f.reg, f.host("bot", "eleven"), "the in-scope box must mint it") |
| 540 | } |
| 541 | |
| 542 | // TestSidecarReconcileDryRunWritesNothing mirrors the instance pass's §4.3 |
| 543 | // mitigation: a pass that reports is a pass that writes nothing at all. |
| 544 | func TestSidecarReconcileDryRunWritesNothing(t *testing.T) { |
| 545 | f := sidecarSetup(t) |
| 546 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}}) |
| 547 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"}) |
| 548 | gone := f.host("gone", "eleven") |
| 549 | f.mustUpsert(t, registry.Route{Host: gone, Kind: "sidecar", Slug: "gone", |
| 550 | App: "eleven", Account: f.sqid, Provisioned: true}) |
| 551 | |
| 552 | f.reconcile(t, ReconcileConfig{DryRun: true}) |
| 553 | |
| 554 | noRoute(t, f.reg, f.host("bot", "eleven"), "dry-run minted a route") |
| 555 | hasRoute(t, f.reg, gone, "dry-run removed a route") |
| 556 | } |
| 557 | |
| 558 | // TestSidecarReconcileLeavesAForeignRowAlone: a row this pass does not own at |
| 559 | // a host a record names is left exactly as alone as before. There is no |
| 560 | // adoption path for sidecars — the suffix is reserved and refused at every |
| 561 | // door, so nothing but this pass can have minted one — and adoption is an |
| 562 | // authority transfer, never a shape coincidence. |
| 563 | func TestSidecarReconcileLeavesAForeignRowAlone(t *testing.T) { |
| 564 | f := sidecarSetup(t) |
| 565 | host := f.host("bot", "eleven") |
| 566 | f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}}) |
| 567 | f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"}) |
| 568 | f.mustUpsert(t, registry.Route{Host: host, Kind: "sidecar", Slug: "bot", |
| 569 | App: "eleven", Account: f.sqid, Args: `["hand","written"]`}) |
| 570 | |
| 571 | f.reconcile(t, ReconcileConfig{}) |
| 572 | |
| 573 | rt := hasRoute(t, f.reg, host, "route gone") |
| 574 | if rt.Provisioned { |
| 575 | t.Fatal("an un-provisioned row must not be adopted by this pass — adoption grants removal authority") |
| 576 | } |
| 577 | if rt.Args != `["hand","written"]` { |
| 578 | t.Fatalf("Args = %q — a row this pass does not own must be left alone", rt.Args) |
| 579 | } |
| 580 | } |
| 581 | |
amadan