amadan
Merge branch 'worktree-instance-config' into main
14 files changed,
+2572
−59
README.md+28 −0internal/appconfig/appconfig.go+6 −1internal/appconfig/awaitingrestart_test.go+100 −0internal/appconfig/configack.go+137 −1internal/appconfig/configack_test.go+149 −0internal/appconfig/environments.go+1 −1internal/appconfig/instances.go+348 −0internal/appconfig/instances_test.go+1412 −0internal/appconfig/routefingerprint.go+8 −0internal/appconfig/store.go+34 −19internal/appconfig/sync.go+110 −10internal/appconfig/sync_test.go+3 −3internal/appconfig/syncexec.go+166 −23internal/appconfig/syncexec_test.go+70 −1
CI passed — run details
diff --git a/README.md b/README.md| index e8c1c817a..0386f2718 100644 |
| --- a/README.md |
| +++ b/README.md |
| @@ -257,6 +257,34 @@ sender record (design spec `2026-09-15-tenant-email-senders-design.md`). |
| drops the database's `mail_counts` table. do not set it until the |
| release after this one has been live for a while. |
| +### Tenant email senders, plan 4 of 5 (2026-09-22) |
| + |
| +This release adds the per-instance config layer (design spec |
| +`2026-09-15-tenant-email-senders-design.md` §6.2): a third config bundle, |
| +layered over an app's default and named-environment config, scoped to one |
| +route's host rather than the whole app. |
| + |
| +- a new failure mode comes with this layer. every config sweep now also |
| + lists `apps/<acct>/<app>/instances-config/` before it converges that |
| + app's routes. s3 throttling, an iam policy change that adds a prefix |
| + condition to `s3:ListBucket`, or a transient 503 on that call all look |
| + the same from a box: the listing fails, and every route of that app |
| + freezes on its last-good config file until the listing succeeds again. |
| + this holds for the whole app, not just routes with an overlay of their |
| + own, because the listing runs once per app regardless of who uses the |
| + layer. |
| +- grep the box's log for `instance config unresolved` to find it. it |
| + prints once per app, naming the account, the app and one route the pass |
| + reached. every other route of that app is covered by the same line and |
| + the same cause. |
| +- it clears itself. once the listing succeeds again, the next pass |
| + converges normally, with nothing to clean up by hand. |
| +- the in-repo iam policies grant `s3:ListBucket` on the whole deployment |
| + bucket with no prefix condition, so a platform box is unaffected by this |
| + on its own account's policy. check a hand-edited or self-hosted |
| + deployment's policy before this rolls there, since a narrower grant is |
| + exactly what would trip it. |
| + |
| ## App config & secrets |
| Per-app config lives in the deployment bucket |
diff --git a/internal/appconfig/appconfig.go b/internal/appconfig/appconfig.go| index 139f12ce5..ff5d13439 100644 |
| --- a/internal/appconfig/appconfig.go |
| +++ b/internal/appconfig/appconfig.go |
| @@ -7,7 +7,12 @@ |
| // Bucket layout (deployment bucket, account-scoped like every other app key |
| // internal/release writes): |
| // |
| -// apps/<acct>/<app>/config.json CAS-guarded config manifest |
| +// apps/<acct>/<app>/config.json CAS-guarded config manifest |
| +// apps/<acct>/<app>/environments/<name>.json named environment overlay |
| +// apps/<acct>/<app>/instances-config/<host>.json one instance's overlay |
| +// |
| +// The three resolve default < environment < instance, per key, and only a |
| +// caller that names a host gets the third (LoadMergedHost, instances.go). |
| package appconfig |
| import ( |
diff --git a/internal/appconfig/awaitingrestart_test.go b/internal/appconfig/awaitingrestart_test.go| index 501ed1d92..ea3e750d0 100644 |
| --- a/internal/appconfig/awaitingrestart_test.go |
| +++ b/internal/appconfig/awaitingrestart_test.go |
| @@ -237,6 +237,106 @@ func TestAwaitingRestartUnknownExecDeliveryIsOff(t *testing.T) { |
| } |
| } |
| +// TestAwaitingRestartInstanceOverlayShadowsADefaultWrite: the third layer |
| +// shadows a default-bundle write the same way a named bundle does, and for |
| +// the same reason. The merged manifest does not move, so syncExecHost's |
| +// bytes.Equal writes nothing and there is no file for anyone to restart over. |
| +func TestAwaitingRestartInstanceOverlayShadowsADefaultWrite(t *testing.T) { |
| + ctx, s, _, _, _, _ := execKit(t) |
| + if err := SetExecDelivery(ctx, s, "bab", "relay", ExecDeliveryFile); err != nil { |
| + t.Fatalf("SetExecDelivery: %v", err) |
| + } |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "relay", "connect.example.com", |
| + map[string]Var{"OWN_KEY": {Kind: "env", Value: "instance"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + routes := []registry.Route{addrRoute("connect.example.com", "relay", "bab", "")} |
| + |
| + if got := AwaitingRestart(ctx, s, routes, "bab", "relay", "", []string{"OWN_KEY"}); !got.Empty() { |
| + t.Fatalf("got %+v, want nothing: OWN_KEY is shadowed by this instance's own layer", got) |
| + } |
| + got := AwaitingRestart(ctx, s, routes, "bab", "relay", "", []string{"OTHER_KEY"}) |
| + if len(got.Hosts) != 1 || got.Hosts[0].Host != "connect.example.com" { |
| + t.Fatalf("Hosts = %+v, want the route for a key its overlay does not state", got.Hosts) |
| + } |
| + got = AwaitingRestart(ctx, s, routes, "bab", "relay", "", []string{"OWN_KEY", "OTHER_KEY"}) |
| + if len(got.Hosts) != 1 { |
| + t.Fatalf("Hosts = %+v, want the route when any key is unshadowed", got.Hosts) |
| + } |
| +} |
| + |
| +// TestAwaitingRestartInstanceOverlayShadowsANamedBundleWrite is the case |
| +// that did not exist before the third layer: a write to the route's OWN |
| +// bound bundle used to be unshadowable, because nothing sat above it. |
| +// Something does now. |
| +func TestAwaitingRestartInstanceOverlayShadowsANamedBundleWrite(t *testing.T) { |
| + ctx, s, _, _, _, _ := execKit(t) |
| + if err := SetExecDelivery(ctx, s, "bab", "relay", ExecDeliveryFile); err != nil { |
| + t.Fatalf("SetExecDelivery: %v", err) |
| + } |
| + if _, err := ApplyEnvironment(ctx, s, "bab", "relay", "prod", |
| + map[string]Var{"K": {Kind: "env", Value: "environment"}}, nil); err != nil { |
| + t.Fatalf("ApplyEnvironment: %v", err) |
| + } |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "relay", "connect.example.com", |
| + map[string]Var{"K": {Kind: "env", Value: "instance"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + routes := []registry.Route{addrRoute("connect.example.com", "relay", "bab", "prod")} |
| + |
| + if got := AwaitingRestart(ctx, s, routes, "bab", "relay", "prod", []string{"K"}); !got.Empty() { |
| + t.Fatalf("got %+v, want nothing: K is shadowed by this instance's own layer", got) |
| + } |
| + if got := AwaitingRestart(ctx, s, routes, "bab", "relay", "prod", []string{"OTHER"}); len(got.Hosts) != 1 { |
| + t.Fatalf("Hosts = %+v, want the route for an unshadowed key", got.Hosts) |
| + } |
| +} |
| + |
| +// TestAwaitingRestartUnresolvedWhenTheOverlayCannotBeRead: a read this |
| +// function could not make is not a determination, on this layer as on the |
| +// other two — the route is named as unresolved rather than reported either |
| +// way. |
| +func TestAwaitingRestartUnresolvedWhenTheOverlayCannotBeRead(t *testing.T) { |
| + ctx, s, _, _, _, _ := execKit(t) |
| + if err := SetExecDelivery(ctx, s, "bab", "relay", ExecDeliveryFile); err != nil { |
| + t.Fatalf("SetExecDelivery: %v", err) |
| + } |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "relay", "connect.example.com", |
| + map[string]Var{"OWN_KEY": {Kind: "env", Value: "instance"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + blip := &failingKeyStore{Store: s, failKey: InstanceConfigKey("bab", "relay", "connect.example.com"), |
| + err: errors.New("s3: transient 503 SlowDown")} |
| + routes := []registry.Route{addrRoute("connect.example.com", "relay", "bab", "")} |
| + got := AwaitingRestart(ctx, blip, routes, "bab", "relay", "", []string{"OWN_KEY"}) |
| + if len(got.Hosts) != 0 || len(got.Unresolved) != 1 || got.Unresolved[0] != "connect.example.com" { |
| + t.Fatalf("got %+v, want the host as unresolved and nothing claimed either way", got) |
| + } |
| +} |
| + |
| +// TestAwaitingRestartEmptyKeysNeverConsultsTheOverlay pins that an empty |
| +// keys (a write with no vars, e.g. flipping exec_delivery on) never asks the |
| +// instance layer anything: there is nothing for it to shadow, per keys' own |
| +// doc comment, so a blip reading it must not downgrade a route this call was |
| +// certain about into Unresolved over a question it was never going to ask. |
| +func TestAwaitingRestartEmptyKeysNeverConsultsTheOverlay(t *testing.T) { |
| + ctx, s, _, _, _, _ := execKit(t) |
| + if err := SetExecDelivery(ctx, s, "bab", "relay", ExecDeliveryFile); err != nil { |
| + t.Fatalf("SetExecDelivery: %v", err) |
| + } |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "relay", "connect.example.com", |
| + map[string]Var{"OWN_KEY": {Kind: "env", Value: "instance"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + blip := &failingKeyStore{Store: s, failKey: InstanceConfigKey("bab", "relay", "connect.example.com"), |
| + err: errors.New("s3: transient 503 SlowDown")} |
| + routes := []registry.Route{addrRoute("connect.example.com", "relay", "bab", "")} |
| + got := AwaitingRestart(ctx, blip, routes, "bab", "relay", "", nil) |
| + if len(got.Hosts) != 1 || got.Hosts[0].Host != "connect.example.com" || len(got.Unresolved) != 0 { |
| + t.Fatalf("got %+v, want the route named in Hosts: an unreadable overlay is irrelevant when keys is empty", got) |
| + } |
| +} |
| + |
| // TestAwaitingRestartSorted pins determinism: registry order is not stable, |
| // and a notice whose lines reshuffle between identical runs is one people |
| // stop reading. |
diff --git a/internal/appconfig/configack.go b/internal/appconfig/configack.go| index eb094d0af..36ccdc14d 100644 |
| --- a/internal/appconfig/configack.go |
| +++ b/internal/appconfig/configack.go |
| @@ -23,7 +23,9 @@ import ( |
| "encoding/json" |
| "errors" |
| "fmt" |
| + "log" |
| "os" |
| + "strings" |
| "time" |
| "github.com/carlosframework/platform/internal/registry" |
| @@ -31,11 +33,43 @@ import ( |
| "github.com/carlosframework/platform/internal/release" |
| ) |
| +// ConfigAckSchema is the ack schema THIS binary writes. It exists so a |
| +// reader can tell an agent that renders the instance config layer (senders |
| +// design §6.2) from one that does not, and the only honest way to do that |
| +// is a positive statement: an ack written by an older agent decodes with |
| +// Schema 0, because the field was not there to write. |
| +// |
| +// It is not the manifest schema and is not tied to it. Bump it when an |
| +// agent starts stating something a reader may need to depend on. |
| +const ConfigAckSchema = 1 |
| + |
| +// instanceConfigETagSchema is the ack schema at which InstanceConfigETag |
| +// first became a field an agent could write — 1, the schema this plan |
| +// introduced it at. InstanceConfigDelivery compares against THIS constant, |
| +// not against ConfigAckSchema: the two happen to be equal today, but they |
| +// answer different questions, and conflating them is a trap the next schema |
| +// bump would fall into silently. ConfigAckSchema is "what schema does this |
| +// BINARY write" and is meant to climb every time the ack states something |
| +// new; instanceConfigETagSchema is "at what schema did THIS field become |
| +// trustworthy" and has no reason to climb again unless the field itself is |
| +// ever redefined. A bump to ConfigAckSchema for some unrelated addition |
| +// (say, schema 2 adds a field about something else entirely) must not |
| +// demote every schema-1 ack — which carries a perfectly good |
| +// InstanceConfigETag — to InstanceConfigUnknownAgent for a question it can |
| +// already answer. |
| +const instanceConfigETagSchema = 1 |
| + |
| // ConfigAck is one route's config-adoption acknowledgement, written after |
| // config sync converges its env/config file — see ConfigAckKey for where. |
| type ConfigAck struct { |
| Box string `json:"box"` |
| Host string `json:"host"` |
| + // Schema is ConfigAckSchema as written by the agent that produced this |
| + // ack, and 0 for every ack written before this field existed. A reader |
| + // asking "does this box render the instance config layer" reads this and |
| + // nothing else: absence of an ack, or an ack at schema 0, both mean the |
| + // question cannot be answered from the box's own statements yet. |
| + Schema int `json:"schema"` |
| // Shape is one of "unit", "socket-exec" or "tcp-exec-file" — spec §5's |
| // three delivery shapes. |
| Shape string `json:"shape"` |
| @@ -61,13 +95,115 @@ type ConfigAck struct { |
| // this pass rendered into CARLOS_RELAY, per region — nil if the route's |
| // manifest carries no CARLOS_RELAY entry at all. |
| RelaySuffixes map[string]string `json:"relay_suffixes,omitempty"` |
| + // InstanceConfigETag is the etag of the instance overlay |
| + // (apps/<acct>/<app>/instances-config/<host>.json) this pass rendered |
| + // into the file, empty when the route has no overlay at all. The writer |
| + // keeps the etag its own Put returned and compares the two |
| + // (InstanceConfigDelivery): equal is delivered, different is stale. A |
| + // map of delivered var names could not make that distinction — an agent |
| + // too old to read the layer and a new agent rendering an overlay whose |
| + // vars the reader does not recognise produce the same map. |
| + InstanceConfigETag string `json:"instance_config_etag,omitempty"` |
| } |
| +// ConfigAckPrefix is where every route's config-adoption ack lives, one |
| +// object per (box, route host). |
| +const ConfigAckPrefix = "control/config-ack/" |
| + |
| // ConfigAckKey is where a route's config-adoption ack lives: additive, one |
| // object per (box, route host), rewritten (never appended to) on every sync |
| // that changes what was delivered. |
| func ConfigAckKey(box, host string) string { |
| - return "control/config-ack/" + box + "/" + host + ".json" |
| + return ConfigAckPrefix + box + "/" + host + ".json" |
| +} |
| + |
| +// The four answers to "is this instance running the overlay I wrote", named |
| +// because three of them are not "no" and a caller that flattened them would |
| +// tell an operator to roll a box that is merely one pass behind. |
| +const ( |
| + // InstanceConfigDelivered: the agent rendered exactly this version of |
| + // the overlay. a route with no overlay, whose caller expects none, is |
| + // also delivered. there is nothing outstanding. |
| + InstanceConfigDelivered = "delivered" |
| + // InstanceConfigStale: the agent rendered a different version. one more |
| + // pass, or a generation that never got read, is the usual cause. |
| + InstanceConfigStale = "stale" |
| + // InstanceConfigUnknownAgent: this box's agent predates the instance |
| + // config layer, so its ack says nothing about it. not "no". the answer |
| + // cannot be had until the box is rolled. |
| + InstanceConfigUnknownAgent = "unknown-agent" |
| + // InstanceConfigNotYetDelivered: no ack for this host at all. acks are |
| + // change-only, so this is the honest answer for a route whose file has |
| + // not moved since its agent rolled. it never means "old agent": that |
| + // question is the box record's agent version, which step 5 reads |
| + // (design §6.5). |
| + InstanceConfigNotYetDelivered = "not-yet-delivered" |
| +) |
| + |
| +// InstanceConfigDelivery compares an ack with the etag a write returned and |
| +// says which of the four states the route is in. want is the etag of the |
| +// overlay the caller wrote, or "" when the caller wrote none. |
| +// |
| +// the comparison is on etags rather than on delivered var names because only |
| +// etags are decidable: an agent too old to read the layer and a new agent |
| +// rendering an overlay whose vars the reader does not recognise deliver the |
| +// same set of names (design §6.5). |
| +// |
| +// The schema test below is against instanceConfigETagSchema, deliberately |
| +// NOT ConfigAckSchema — see that constant's own doc comment for the hazard |
| +// a future schema bump would otherwise walk into. |
| +func InstanceConfigDelivery(a *ConfigAck, want string) string { |
| + switch { |
| + case a == nil: |
| + return InstanceConfigNotYetDelivered |
| + case a.Schema < instanceConfigETagSchema: |
| + return InstanceConfigUnknownAgent |
| + case a.InstanceConfigETag == want: |
| + return InstanceConfigDelivered |
| + default: |
| + return InstanceConfigStale |
| + } |
| +} |
| + |
| +// LatestConfigAck reads the freshest ack naming host, across every box that |
| +// has written one, and nil (with no error) when nothing has. a route can |
| +// legitimately have acks from two boxes at once across a migration, and the |
| +// freshest is the one describing what is running now. |
| +// |
| +// it lists the whole ack prefix, which is the same limit relayConfigAcks |
| +// accepts (internal/console/relayops.go): there is no per-host box list to |
| +// narrow a listing by. the Gets are narrowed, though. only keys whose |
| +// basename is this host are read. |
| +func LatestConfigAck(ctx context.Context, s release.Store, host string) (*ConfigAck, error) { |
| + keys, err := s.List(ctx, ConfigAckPrefix) |
| + if err != nil { |
| + return nil, err |
| + } |
| + var best *ConfigAck |
| + for _, key := range keys { |
| + if !strings.HasSuffix(key, "/"+host+".json") { |
| + continue |
| + } |
| + body, _, err := s.Get(ctx, key) |
| + if err != nil { |
| + if errors.Is(err, os.ErrNotExist) { |
| + continue |
| + } |
| + return nil, err |
| + } |
| + var a ConfigAck |
| + if err := json.Unmarshal(body, &a); err != nil { |
| + // one unreadable object must not hide the box that answered |
| + // properly, and there is nothing in it to report either way. |
| + log.Printf("config: config ack %s: %v", key, err) |
| + continue |
| + } |
| + if best == nil || a.At.After(best.At) { |
| + latest := a |
| + best = &latest |
| + } |
| + } |
| + return best, nil |
| } |
| // WriteConfigAck writes ack, get-then-put with the CAS etag of whatever is |
diff --git a/internal/appconfig/configack_test.go b/internal/appconfig/configack_test.go| index 9a8a419b5..6725777ec 100644 |
| --- a/internal/appconfig/configack_test.go |
| +++ b/internal/appconfig/configack_test.go |
| @@ -6,6 +6,7 @@ import ( |
| "errors" |
| "strings" |
| "testing" |
| + "time" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/relay" |
| @@ -227,3 +228,151 @@ func TestSyncAckHibernatingRouteReportsNoRestart(t *testing.T) { |
| t.Fatalf("ack.RestartError = %q, want empty", got.RestartError) |
| } |
| } |
| + |
| +// The ack says which version of the instance layer the agent rendered |
| +// (design §6.5). The console holds the etag its own write returned and |
| +// compares the two, which is decidable in a way a map of delivered key |
| +// names is not: an old agent and a new agent rendering an overlay that sets |
| +// nothing look identical by key, and not at all by etag. |
| +func TestConfigAckNamesTheInstanceOverlayItRendered(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + reg := testRegistry(t) |
| + envDir := t.TempDir() |
| + priv, _ := syncRelayTestKeys(t) |
| + |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + _, want, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil) |
| + if err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + for _, host := range []string{"book.example.com", "cart.example.com"} { |
| + if err := reg.Upsert(registry.Route{ |
| + Host: host, Kind: "instance", App: "titogo", Account: "bab", Status: "ready", |
| + Backing: registry.BackingUnit, |
| + }); err != nil { |
| + t.Fatalf("Upsert %s: %v", host, err) |
| + } |
| + } |
| + acks := map[string]ConfigAck{} |
| + if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, |
| + func(a ConfigAck) { acks[a.Host] = a }); err != nil { |
| + t.Fatalf("Sync: %v", err) |
| + } |
| + book, ok := acks["book.example.com"] |
| + if !ok { |
| + t.Fatal("no ack for the overlaid route") |
| + } |
| + if book.Schema != ConfigAckSchema { |
| + t.Fatalf("Schema = %d, want %d — absence is how an older agent is recognised", book.Schema, ConfigAckSchema) |
| + } |
| + if book.InstanceConfigETag != want { |
| + t.Fatalf("InstanceConfigETag = %q, want the etag the write returned %q", book.InstanceConfigETag, want) |
| + } |
| + cart, ok := acks["cart.example.com"] |
| + if !ok { |
| + t.Fatal("no ack for the sibling") |
| + } |
| + if cart.Schema != ConfigAckSchema || cart.InstanceConfigETag != "" { |
| + t.Fatalf("sibling ack = %+v, want the schema and an empty etag: it has no overlay to render", cart) |
| + } |
| +} |
| + |
| +// The same statement from the exec half, which is the half an instance |
| +// sender's box actually runs (design §6.4's delivery check). |
| +func TestSyncExecConfigAckNamesTheInstanceOverlayItRendered(t *testing.T) { |
| + ctx, s, reg, dd, privB64, _ := execKit(t) |
| + priv, err := ParsePrivateKey(privB64, "test") |
| + if err != nil { |
| + t.Fatalf("ParsePrivateKey: %v", err) |
| + } |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + _, want, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil) |
| + if err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + if err := reg.Upsert(registry.Route{ |
| + Host: "book.example.com", Kind: "instance", App: "titogo", Account: "bab", |
| + Status: "ready", Socket: "/run/carlos/book.sock", |
| + }); err != nil { |
| + t.Fatalf("Upsert: %v", err) |
| + } |
| + var got []ConfigAck |
| + if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", |
| + func(a ConfigAck) { got = append(got, a) }); err != nil { |
| + t.Fatalf("SyncExec: %v", err) |
| + } |
| + if len(got) != 1 { |
| + t.Fatalf("acks = %+v, want exactly one", got) |
| + } |
| + if got[0].Schema != ConfigAckSchema || got[0].InstanceConfigETag != want { |
| + t.Fatalf("ack = %+v, want schema %d and etag %q", got[0], ConfigAckSchema, want) |
| + } |
| +} |
| + |
| +// The four answers, and they are four rather than two on purpose. "This |
| +// agent is too old to render the layer" and "this agent rendered an older |
| +// version of it" call for different sentences from whoever asks: one is a |
| +// box to roll, the other is a pass to wait for. |
| +func TestInstanceConfigDeliveryVerdicts(t *testing.T) { |
| + cases := []struct { |
| + name string |
| + ack *ConfigAck |
| + want string |
| + got string |
| + }{ |
| + {"not yet acked", nil, "etag-1", InstanceConfigNotYetDelivered}, |
| + {"agent older than the layer", &ConfigAck{Schema: 0, InstanceConfigETag: ""}, "etag-1", InstanceConfigUnknownAgent}, |
| + {"rendered this overlay", &ConfigAck{Schema: ConfigAckSchema, InstanceConfigETag: "etag-1"}, "etag-1", InstanceConfigDelivered}, |
| + {"rendered an older one", &ConfigAck{Schema: ConfigAckSchema, InstanceConfigETag: "etag-0"}, "etag-1", InstanceConfigStale}, |
| + {"no overlay, and none expected", &ConfigAck{Schema: ConfigAckSchema}, "", InstanceConfigDelivered}, |
| + {"no overlay, but one was written", &ConfigAck{Schema: ConfigAckSchema}, "etag-1", InstanceConfigStale}, |
| + } |
| + for _, c := range cases { |
| + if got := InstanceConfigDelivery(c.ack, c.want); got != c.got { |
| + t.Errorf("%s: InstanceConfigDelivery = %q, want %q", c.name, got, c.got) |
| + } |
| + } |
| +} |
| + |
| +// One route can have acks from more than one box across a migration, so the |
| +// reader takes the freshest and ignores every other host's. |
| +func TestLatestConfigAckPicksTheFreshestForTheHost(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + older := ConfigAck{Schema: ConfigAckSchema, Box: "box-1", Host: "book.example.com", Shape: "unit", |
| + At: time.Date(2026, 9, 15, 10, 0, 0, 0, time.UTC), InstanceConfigETag: "etag-0"} |
| + newer := ConfigAck{Schema: ConfigAckSchema, Box: "box-2", Host: "book.example.com", Shape: "unit", |
| + At: time.Date(2026, 9, 15, 12, 0, 0, 0, time.UTC), InstanceConfigETag: "etag-1"} |
| + other := ConfigAck{Schema: ConfigAckSchema, Box: "box-2", Host: "cart.example.com", Shape: "unit", |
| + At: time.Date(2026, 9, 15, 13, 0, 0, 0, time.UTC), InstanceConfigETag: "etag-9"} |
| + for _, a := range []ConfigAck{older, newer, other} { |
| + if err := WriteConfigAck(ctx, s, a); err != nil { |
| + t.Fatalf("WriteConfigAck %s/%s: %v", a.Box, a.Host, err) |
| + } |
| + } |
| + got, err := LatestConfigAck(ctx, s, "book.example.com") |
| + if err != nil || got == nil { |
| + t.Fatalf("LatestConfigAck = %+v %v", got, err) |
| + } |
| + if got.Box != "box-2" || got.InstanceConfigETag != "etag-1" { |
| + t.Fatalf("LatestConfigAck = %+v, want box-2's etag-1", got) |
| + } |
| + if v := InstanceConfigDelivery(got, "etag-1"); v != InstanceConfigDelivered { |
| + t.Fatalf("delivery = %q, want %q", v, InstanceConfigDelivered) |
| + } |
| + |
| + none, err := LatestConfigAck(ctx, s, "desk.example.com") |
| + if err != nil || none != nil { |
| + t.Fatalf("LatestConfigAck for a host with no ack = %+v %v, want (nil, nil)", none, err) |
| + } |
| + if v := InstanceConfigDelivery(none, "etag-1"); v != InstanceConfigNotYetDelivered { |
| + t.Fatalf("delivery = %q, want %q", v, InstanceConfigNotYetDelivered) |
| + } |
| +} |
diff --git a/internal/appconfig/environments.go b/internal/appconfig/environments.go| index c5d376921..b3b4a4719 100644 |
| --- a/internal/appconfig/environments.go |
| +++ b/internal/appconfig/environments.go |
| @@ -115,7 +115,7 @@ func applyEnvironment(ctx context.Context, s release.Store, acct, app, envName s |
| if !ValidEnvironmentName(envName) { |
| return 0, fmt.Errorf("config: invalid environment name %q", envName) |
| } |
| - removed, err := applyAt(ctx, s, EnvironmentKey(acct, app, envName), app, envName, sets, unsets) |
| + removed, _, err := applyAt(ctx, s, EnvironmentKey(acct, app, envName), app, envName, sets, unsets) |
| if err != nil { |
| return 0, err |
| } |
diff --git a/internal/appconfig/instances.go b/internal/appconfig/instances.go| new file mode 100644 |
| index 000000000..686c4b29e |
| --- /dev/null |
| +++ b/internal/appconfig/instances.go |
| @@ -0,0 +1,348 @@ |
| +// The instance config layer (tenant-email senders design §6.2): a third |
| +// bundle, one object per route host, resolved after the named environment. |
| +// Same Manifest shape, same sealed envelopes, same name and value rules as |
| +// the other two — a per-instance bundle is not a new format, just a third |
| +// bucket key the same shape is written to. |
| +// |
| +// Bucket layout addition: |
| +// |
| +// apps/<acct>/<app>/instances-config/<host>.json CAS-guarded, schema-1 manifest |
| +// |
| +// It exists so a credential can be scoped to ONE instance. #148's shape is |
| +// one instance per customer under a single app, and everything the platform |
| +// delivers today lands in a bundle every instance of the app renders, so a |
| +// per-customer secret delivered through the default bundle reaches every |
| +// other customer's process. |
| +// |
| +// Manifest.ExecDelivery is NOT read from this layer. That is the ruling at |
| +// appconfig.go's ExecDelivery doc comment, unchanged: a per-route delivery |
| +// flag would need a registry column, a record field and reconcile |
| +// convergence, and two routes resolving through the same bundle could |
| +// disagree about whether they are delivered. A per-host bundle is exactly |
| +// the granularity that ruling refused, so resolveExecDelivery is never |
| +// handed one of these manifests and a field that somehow appears in one is |
| +// inert. |
| +package appconfig |
| + |
| +import ( |
| + "context" |
| + "fmt" |
| + "regexp" |
| + "strings" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// InstancesConfigPrefix is the list prefix behind "which instances of this |
| +// app have an overlay" — every instances-config/<host>.json key shares it. |
| +// Inside apps/<acct>/<app>/ on purpose: release.PurgeApp's prefix sweep |
| +// reaches it, so a purged app cannot leave an object full of sealed |
| +// secrets behind. |
| +func InstancesConfigPrefix(acct, app string) string { |
| + return release.AppPrefix(acct, app) + "instances-config/" |
| +} |
| + |
| +// InstanceConfigKey is the bucket key of one instance's overlay. host is a |
| +// route's Host column verbatim, which is what makes the key derivable from |
| +// a registry row with nothing to look up. |
| +func InstanceConfigKey(acct, app, host string) string { |
| + return InstancesConfigPrefix(acct, app) + host + ".json" |
| +} |
| + |
| +// instanceHostRe is a bare lowercase hostname: labels of letters, digits |
| +// and hyphens, joined by dots, never starting or ending with a hyphen or a |
| +// dot. Case-restricted like ValidEnvironmentName and for the same reason, |
| +// with one more: this string is also the file name sync derives on the box |
| +// (sync.go's filepath.Base guard), so one convention keeps a bucket key and |
| +// a file name from disagreeing about which host they belong to. |
| +var instanceHostRe = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$`) |
| + |
| +// ValidInstanceHost reports whether host is usable as an instance overlay |
| +// key. A separator or a traversal is refused at the one gate every caller |
| +// passes through, because this string is concatenated into a bucket key: a |
| +// host of "../config" would otherwise name the app's default bundle. |
| +func ValidInstanceHost(host string) bool { |
| + if host == "" || len(host) > 253 || strings.Contains(host, "..") { |
| + return false |
| + } |
| + // Per-label bound as well as the whole-name one, matching DNS and |
| + // launchctx/controlname's label rule. Without it a single 200-character |
| + // label passes here and is refused everywhere else that handles the same |
| + // host, which is a disagreement between two gates rather than one rule. |
| + for _, label := range strings.Split(host, ".") { |
| + if len(label) > 63 { |
| + return false |
| + } |
| + } |
| + return instanceHostRe.MatchString(host) |
| +} |
| + |
| +// LoadInstanceConfig reads one instance's overlay, returning the manifest, |
| +// the etag it was read at, and whether the object exists at all. |
| +// |
| +// The etag is the point of this signature. The delivery ack (design §6.5) |
| +// carries the etag the agent actually rendered, and the console compares it |
| +// with the one its own Put returned: that comparison is decidable, where a |
| +// map of delivered key names cannot tell an old agent apart from a new one |
| +// rendering a route with no overlay vars. |
| +// |
| +// Not-found is (nil, "", false, nil): an instance with no overlay is the |
| +// normal case and the state every instance is in until something writes one. |
| +func LoadInstanceConfig(ctx context.Context, s release.Store, acct, app, host string) (*Manifest, string, bool, error) { |
| + if !release.ValidAccount(acct) { |
| + return nil, "", false, fmt.Errorf("config: invalid account %q", acct) |
| + } |
| + if !release.ValidApp(app) { |
| + return nil, "", false, fmt.Errorf("config: invalid app %q", app) |
| + } |
| + if !ValidInstanceHost(host) { |
| + return nil, "", false, fmt.Errorf("config: invalid instance host %q", host) |
| + } |
| + return getIfExistsAtETag(ctx, s, InstanceConfigKey(acct, app, host)) |
| +} |
| + |
| +// ApplyInstance writes sets and removes unsets from ONE instance's overlay |
| +// in one CAS write, returning how many of unsets were actually present and |
| +// THE ETAG IT WROTE AT. |
| +// |
| +// That etag is the contract with the box. The console keeps it, the agent |
| +// reports the etag it rendered in its config ack, and the two are compared |
| +// (design §6.5, InstanceConfigDelivery). A map of delivered key names would |
| +// not do: it cannot tell an agent too old to render this layer apart from a |
| +// new agent rendering a route whose overlay sets nothing it recognises. |
| +// |
| +// There is no CLI verb and no console door onto this function, by design |
| +// (§6.2, "only the platform writes this layer"). It exists so a credential |
| +// can be scoped to one instance, and the only caller that scopes one is the |
| +// email walk in step 5. |
| +// |
| +// Unset-only against an overlay that does not exist is refused, not turned |
| +// into a create, for the reason applyAt's own doc comment gives: an empty |
| +// manifest that EXISTS is a state the sweeps converge a file to, and |
| +// minting one to hold nothing would deliver an empty file to a live |
| +// instance. |
| +func ApplyInstance(ctx context.Context, s release.Store, acct, app, host string, sets map[string]Var, unsets []string) (int, string, error) { |
| + if !release.ValidAccount(acct) { |
| + return 0, "", fmt.Errorf("config: invalid account %q", acct) |
| + } |
| + if !release.ValidApp(app) { |
| + return 0, "", fmt.Errorf("config: invalid app %q", app) |
| + } |
| + if !ValidInstanceHost(host) { |
| + return 0, "", fmt.Errorf("config: invalid instance host %q", host) |
| + } |
| + removed, etag, err := applyAt(ctx, s, InstanceConfigKey(acct, app, host), app, "instance "+host, sets, unsets) |
| + if err != nil { |
| + return 0, "", err |
| + } |
| + // The generation object is the ONLY signal this layer has. |
| + // ConfigRouteFingerprints (routefingerprint.go) digests registry columns |
| + // and cannot see a bucket-side change, so an overlay written without |
| + // this touch would wait out configSyncLoop's hourly floor. Same |
| + // best-effort demotion every other write here makes: the manifest has |
| + // landed, so a failed touch is logged, never returned. |
| + touchLogged(ctx, s, acct, app) |
| + return removed, etag, nil |
| +} |
| + |
| +// DeleteInstanceConfig removes one instance's overlay outright, which |
| +// converges that route back to its group's merge on the next pass. Absent |
| +// is success: a retried teardown must converge rather than fail, and |
| +// release.Store.Delete is already idempotent. |
| +// |
| +// The generation is touched even when nothing was removed. Branching on it |
| +// would mean a Get before every delete to find out, and one redundant touch |
| +// costs a tiny object write where a missed one costs an hour of a live |
| +// instance holding config nothing meant it to have. |
| +func DeleteInstanceConfig(ctx context.Context, s release.Store, acct, app, host string) error { |
| + if !release.ValidAccount(acct) { |
| + return fmt.Errorf("config: invalid account %q", acct) |
| + } |
| + if !release.ValidApp(app) { |
| + return fmt.Errorf("config: invalid app %q", app) |
| + } |
| + if !ValidInstanceHost(host) { |
| + return fmt.Errorf("config: invalid instance host %q", host) |
| + } |
| + if err := s.Delete(ctx, InstanceConfigKey(acct, app, host)); err != nil { |
| + return err |
| + } |
| + touchLogged(ctx, s, acct, app) |
| + return nil |
| +} |
| + |
| +// LoadMergedHost is LoadMerged plus the instance layer: default < |
| +// environment < instance, the instance winning per key, and the overlay's |
| +// etag returned alongside so a caller can check delivery against it later |
| +// (design §6.2, §6.5). An empty etag means this host has no overlay, which |
| +// is a different statement from an agent that never rendered one. |
| +// |
| +// LoadMerged keeps its signature and its meaning. Everything that asks |
| +// "what does this APP's environment resolve to" still gets two layers, and |
| +// only a caller that names a host gets three. |
| +// |
| +// The two sync sweeps do NOT call this function, deliberately: they resolve |
| +// one group base for every route that shares it and would re-fetch |
| +// config.json per route if they came through here. They reach the same |
| +// answer through instanceOverlays and applyInstanceOverlay instead, and the |
| +// equivalence for a VALID host is pinned by a test rather than by hope — |
| +// TestSyncRendersTheInstanceOverlayForItsOwnRouteOnly compares a sweep's |
| +// rendered file against this function's own answer for the same host. |
| +// |
| +// That equivalence does NOT extend to a host that fails ValidInstanceHost. |
| +// This function refuses one outright (see the guard below): a hostname a |
| +// caller cannot even ask about is an error here. The sweeps never make that |
| +// call at all — instanceOverlayLoader treats such a host as "not named by |
| +// the listing", which reads as "no overlay", and the route converges on its |
| +// group's merge same as any other unconfigured instance. A caller that |
| +// wants the sweep's own answer for a host it cannot yet prove valid (step |
| +// 5's write path, before ApplyInstance's own ValidInstanceHost gate runs) |
| +// must not substitute this function for that reason: the two disagree here, |
| +// on purpose, and only one of them is what a box will actually deliver. |
| +func LoadMergedHost(ctx context.Context, s release.Store, acct, app, envName, host string) (*Manifest, string, error) { |
| + if !ValidInstanceHost(host) { |
| + return nil, "", fmt.Errorf("config: invalid instance host %q", host) |
| + } |
| + base, err := LoadMerged(ctx, s, acct, app, envName) |
| + if err != nil { |
| + return nil, "", err |
| + } |
| + inst, etag, found, err := LoadInstanceConfig(ctx, s, acct, app, host) |
| + if err != nil { |
| + return nil, "", err |
| + } |
| + if !found { |
| + return base, "", nil |
| + } |
| + return applyInstanceOverlay(app, base, inst), etag, nil |
| +} |
| + |
| +// applyInstanceOverlay lays inst over base, per key. base may be nil (the |
| +// app has no default bundle and the route is unbound), in which case the |
| +// merged view is the overlay ALONE — which the callers treat as the |
| +// wholesale replacement it is, see sync.go's defaultMissing guard. |
| +// |
| +// A nil overlay returns base untouched, pointer and all: that is the path |
| +// every instance without an overlay takes, and rebuilding the manifest |
| +// there would strip exec_delivery off the default bundle object LoadMerged |
| +// hands back for an unbound route. |
| +// |
| +// The merged view never carries exec_delivery FROM the overlay. See this |
| +// file's own doc comment: a per-host delivery flag is the granularity the |
| +// #195 ruling refused, and the merged manifest is what gets rendered into a |
| +// file, where a delivery flag has no business appearing anyway. |
| +func applyInstanceOverlay(app string, base, inst *Manifest) *Manifest { |
| + if inst == nil { |
| + return base |
| + } |
| + merged := &Manifest{Schema: 1, App: app, Vars: map[string]Var{}} |
| + if base != nil { |
| + for k, v := range base.Vars { |
| + merged.Vars[k] = v |
| + } |
| + } |
| + for k, v := range inst.Vars { |
| + merged.Vars[k] = v |
| + } |
| + return merged |
| +} |
| + |
| +// instanceOverlays lists which of this app's instances have an overlay at |
| +// all: one listing per (account, app), cached by the caller for a whole |
| +// pass, so a sweep pays one List per app and then one Get per instance that |
| +// actually has one. Reading every route's key blind would cost a bucket |
| +// round trip per instance on a box where nothing has an overlay, which is |
| +// every box in the fleet the day this ships. |
| +// |
| +// A key that does not end in .json, or whose host does not survive |
| +// ValidInstanceHost, is skipped rather than reported: nothing can have |
| +// written it through this package, so it is not this app's overlay |
| +// whatever it is. |
| +func instanceOverlays(ctx context.Context, s release.Store, acct, app string) (map[string]bool, error) { |
| + prefix := InstancesConfigPrefix(acct, app) |
| + keys, err := s.List(ctx, prefix) |
| + if err != nil { |
| + return nil, err |
| + } |
| + out := make(map[string]bool, len(keys)) |
| + for _, k := range keys { |
| + host := strings.TrimSuffix(strings.TrimPrefix(k, prefix), ".json") |
| + if host == strings.TrimPrefix(k, prefix) || !ValidInstanceHost(host) { |
| + continue |
| + } |
| + out[host] = true |
| + } |
| + return out, nil |
| +} |
| + |
| +// instanceOverlayListErr marks an error as coming from instanceOverlayLoader's |
| +// cached LISTING (instanceOverlays) rather than from a per-host Get. The |
| +// listing is the one call every route of an app shares, so every route |
| +// fails identically and for the same reason when it errors — a caller may |
| +// dedup on that. A per-host Get or parse failure carries no such guarantee: |
| +// two routes can have two separately corrupt overlay objects, each failing |
| +// for its own reason, and folding both into one line would hide the second |
| +// one's failure entirely. errors.As is how a caller tells the two apart. |
| +type instanceOverlayListErr struct{ error } |
| + |
| +// Unwrap lets errors.Is/errors.As see through the wrapper to the listing |
| +// failure itself, the same courtesy every %w wrap in this package gives. |
| +func (e *instanceOverlayListErr) Unwrap() error { return e.error } |
| + |
| +// instanceOverlayLoader is one pass's instance-layer reader, shared by both |
| +// sweeps (Sync and SyncExec) so neither can drift from the other about what |
| +// a host gets or what it costs. One List per (account, app), cached for the |
| +// pass including its failure, then a Get only for a host that listing |
| +// named. |
| +// |
| +// A listing FAILURE is returned to the caller wrapped in |
| +// instanceOverlayListErr, never folded into "no overlay". The two sweeps |
| +// treat it the way they treat a bundle fetch failure: leave the file |
| +// exactly as it is, say so, retry next pass. Folding it away would silently |
| +// deliver the app-wide merge to an instance that is meant to have its own |
| +// credential, which is the one outcome this layer exists to prevent. |
| +// |
| +// A host the listing named but whose object is gone by the time of the Get |
| +// is simply "no overlay": a delete racing the pass converges the route back |
| +// to its group's merge, which is what the delete asked for. |
| +// |
| +// The index map below has no lock. That's safe only because each call to |
| +// this function returns a fresh closure over its own map, and both Sync and |
| +// SyncExec build one loader per pass and drive it from their own single |
| +// route loop — never two goroutines sharing one closure. A caller that ever |
| +// shared one instanceOverlayLoader value across concurrently running sweeps |
| +// would race on this map; don't. |
| +func instanceOverlayLoader(ctx context.Context, s release.Store) func(acct, app, host string) (*Manifest, string, error) { |
| + type appKey struct{ acct, app string } |
| + type indexRes struct { |
| + hosts map[string]bool |
| + err error |
| + } |
| + index := map[appKey]indexRes{} |
| + return func(acct, app, host string) (*Manifest, string, error) { |
| + ak := appKey{acct, app} |
| + idx, done := index[ak] |
| + if !done { |
| + hosts, err := instanceOverlays(ctx, s, acct, app) |
| + if err != nil { |
| + err = &instanceOverlayListErr{fmt.Errorf("%s/%s: instance config listing failed: %w", acct, app, err)} |
| + } |
| + idx = indexRes{hosts: hosts, err: err} |
| + index[ak] = idx |
| + } |
| + if idx.err != nil { |
| + return nil, "", idx.err |
| + } |
| + if !idx.hosts[host] { |
| + return nil, "", nil |
| + } |
| + m, etag, found, err := LoadInstanceConfig(ctx, s, acct, app, host) |
| + if err != nil { |
| + return nil, "", fmt.Errorf("%s/%s: instance config for %s: %w", acct, app, host, err) |
| + } |
| + if !found { |
| + return nil, "", nil |
| + } |
| + return m, etag, nil |
| + } |
| +} |
diff --git a/internal/appconfig/instances_test.go b/internal/appconfig/instances_test.go| new file mode 100644 |
| index 000000000..beb862c4d |
| --- /dev/null |
| +++ b/internal/appconfig/instances_test.go |
| @@ -0,0 +1,1412 @@ |
| +package appconfig |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "crypto/ecdh" |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "sync" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// putInstanceRaw writes an instance-layer object without going through the |
| +// package's own writer, which does not exist until task 2. Tests in this |
| +// file that only READ the layer use it, so task 1 can be verified on its |
| +// own. |
| +func putInstanceRaw(t *testing.T, ctx context.Context, s release.Store, acct, app, host string, m *Manifest) string { |
| + t.Helper() |
| + body, err := json.MarshalIndent(m, "", " ") |
| + if err != nil { |
| + t.Fatalf("marshal instance manifest: %v", err) |
| + } |
| + etag, err := s.Put(ctx, InstanceConfigKey(acct, app, host), body, "") |
| + if err != nil { |
| + t.Fatalf("put instance manifest: %v", err) |
| + } |
| + return etag |
| +} |
| + |
| +// The layer sits under the app's own prefix, beside config.json and |
| +// environments/, so release.PurgeApp's prefix sweep reaches it and a purged |
| +// app leaves no instance overlay behind holding sealed secrets. |
| +func TestInstanceConfigKeysSitUnderTheAppsOwnPrefix(t *testing.T) { |
| + if got, want := InstancesConfigPrefix("bab", "titogo"), "apps/bab/titogo/instances-config/"; got != want { |
| + t.Fatalf("InstancesConfigPrefix = %q, want %q", got, want) |
| + } |
| + if got, want := InstanceConfigKey("bab", "titogo", "book.customer.example.com"), |
| + "apps/bab/titogo/instances-config/book.customer.example.com.json"; got != want { |
| + t.Fatalf("InstanceConfigKey = %q, want %q", got, want) |
| + } |
| +} |
| + |
| +// A host becomes a bucket key segment here, so the one gate every caller |
| +// passes through is where a separator, a traversal or an empty name is |
| +// refused. The shape is a bare lowercase hostname, the same thing a route |
| +// row's Host column holds and the same thing sync.go's own path guard |
| +// (rt.Host != filepath.Base(rt.Host)) already insists on before it derives |
| +// a file name from one. |
| +func TestValidInstanceHostRefusesAnythingThatIsNotABareHost(t *testing.T) { |
| + for _, bad := range []string{ |
| + "", ".", "..", "a/b", "../etc", "Book.example.com", "book..example.com", |
| + "-lead.example.com", "trail-.example.com", "book.example.com/", |
| + strings.Repeat("a", 254), |
| + } { |
| + if ValidInstanceHost(bad) { |
| + t.Errorf("ValidInstanceHost(%q) = true", bad) |
| + } |
| + } |
| + for _, good := range []string{"book.customer.example.com", "inst1", "a-b.c-d.example.com", "127-0-0-1.example.com"} { |
| + if !ValidInstanceHost(good) { |
| + t.Errorf("ValidInstanceHost(%q) = false", good) |
| + } |
| + } |
| +} |
| + |
| +// The reader answers three ways, and the caller needs all three apart: a |
| +// manifest and the etag it was read at, "no object" without an error (an |
| +// instance with no overlay is the normal case, not a fault), and an error |
| +// for anything else. The etag is what the delivery ack carries (design §6.5), |
| +// so a read that dropped it would leave the agent unable to say which |
| +// version of the layer it rendered. |
| +func TestLoadInstanceConfigReadsTheLayerAndItsETag(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + |
| + m, etag, found, err := LoadInstanceConfig(ctx, s, "bab", "titogo", "book.customer.example.com") |
| + if err != nil || found || m != nil || etag != "" { |
| + t.Fatalf("absent overlay = %+v %q %v %v, want (nil, \"\", false, nil)", m, etag, found, err) |
| + } |
| + |
| + want := putInstanceRaw(t, ctx, s, "bab", "titogo", "book.customer.example.com", |
| + &Manifest{Schema: 1, App: "titogo", Vars: map[string]Var{"TENANT_SMTP_USER": {Kind: "env", Value: "book"}}}) |
| + m, etag, found, err = LoadInstanceConfig(ctx, s, "bab", "titogo", "book.customer.example.com") |
| + if err != nil || !found || m == nil { |
| + t.Fatalf("present overlay = %+v %v %v", m, found, err) |
| + } |
| + if etag == "" || etag != want { |
| + t.Fatalf("etag = %q, want the object's own %q", etag, want) |
| + } |
| + if m.Vars["TENANT_SMTP_USER"].Value != "book" { |
| + t.Fatalf("overlay lost its var: %+v", m.Vars) |
| + } |
| + |
| + // The same newer-schema refusal every other bundle read makes: acting on |
| + // a manifest whose unknown fields json.Unmarshal has already dropped |
| + // would silently discard whatever they meant. |
| + putInstanceRaw(t, ctx, s, "bab", "titogo", "newer.example.com", &Manifest{Schema: 2, App: "titogo"}) |
| + if _, _, _, err := LoadInstanceConfig(ctx, s, "bab", "titogo", "newer.example.com"); err == nil { |
| + t.Fatal("a schema-2 overlay was accepted, want a refusal naming the upgrade") |
| + } |
| + |
| + // A host that is not a bare hostname never reaches the bucket at all. |
| + if _, _, _, err := LoadInstanceConfig(ctx, s, "bab", "titogo", "../config"); err == nil { |
| + t.Fatal("LoadInstanceConfig accepted a traversal host") |
| + } |
| +} |
| + |
| +// The write door. It returns the etag it wrote at, which is the whole |
| +// mechanism behind the delivery check (design §6.5): the caller keeps that |
| +// string and compares it with what an agent's ack says it rendered. |
| +func TestApplyInstanceWritesTheLayerAndReturnsItsETag(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + |
| + removed, etag, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", |
| + map[string]Var{"TENANT_SMTP_USER": {Kind: "env", Value: "book"}}, nil) |
| + if err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + if removed != 0 { |
| + t.Fatalf("removed = %d on a create, want 0", removed) |
| + } |
| + if etag == "" { |
| + t.Fatal("ApplyInstance returned an empty etag, so nothing can check delivery against it") |
| + } |
| + m, readETag, found, err := LoadInstanceConfig(ctx, s, "bab", "titogo", "book.customer.example.com") |
| + if err != nil || !found { |
| + t.Fatalf("LoadInstanceConfig after the write = %+v %v %v", m, found, err) |
| + } |
| + if readETag != etag { |
| + t.Fatalf("the reader saw etag %q, the writer returned %q — the delivery check compares these two", readETag, etag) |
| + } |
| + if m.Schema != 1 || m.App != "titogo" || m.Vars["TENANT_SMTP_USER"].Value != "book" { |
| + t.Fatalf("the written overlay is not a schema-1 manifest of this app: %+v", m) |
| + } |
| + |
| + // A second write moves the etag, which is what makes a stale ack |
| + // detectable at all. |
| + _, etag2, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", |
| + map[string]Var{"TENANT_SMTP_USER": {Kind: "env", Value: "book2"}}, nil) |
| + if err != nil { |
| + t.Fatalf("ApplyInstance (second): %v", err) |
| + } |
| + if etag2 == etag { |
| + t.Fatalf("etag did not move on a changed write: %q", etag2) |
| + } |
| + |
| + // Unset reports how many of the keys were actually there, the same |
| + // contract every other bundle's unset has. |
| + removed, _, err = ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", nil, |
| + []string{"TENANT_SMTP_USER", "NEVER_SET"}) |
| + if err != nil || removed != 1 { |
| + t.Fatalf("unset = %d %v, want 1 and no error", removed, err) |
| + } |
| + |
| + // The var-name and var-value rules are the bundle's, not this layer's: |
| + // applyAt refuses before anything reaches the bucket. |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", |
| + map[string]Var{"not a name": {Kind: "env", Value: "x"}}, nil); err == nil { |
| + t.Fatal("an invalid var name was accepted into the instance layer") |
| + } |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "../config", nil, nil); err == nil { |
| + t.Fatal("ApplyInstance accepted a traversal host") |
| + } |
| +} |
| + |
| +// An unset against an overlay that does not exist is refused rather than |
| +// minting an empty one, the same refusal applyAt already makes for the |
| +// other two bundles and for the same reason: an empty manifest that EXISTS |
| +// means "configured, then fully unset" by this package's contract, which is |
| +// a state the sweeps converge a file to. |
| +func TestApplyInstanceRefusesUnsetOnlyAgainstAMissingOverlay(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", nil, []string{"K"}) |
| + if err == nil { |
| + t.Fatal("unsetting from a nonexistent overlay was turned into a create") |
| + } |
| + if !strings.Contains(err.Error(), "book.customer.example.com") { |
| + t.Fatalf("the refusal does not name the instance: %v", err) |
| + } |
| +} |
| + |
| +// Writing the layer bumps the app's generation object, the way |
| +// SetExecDeliveryAt does. It has to: ConfigRouteFingerprints digests |
| +// registry columns only (routefingerprint.go:58) and cannot see a |
| +// bucket-side change, so without the bump an overlay would wait out the |
| +// agent's hourly floor with nothing to signal it. |
| +func TestWritingTheInstanceLayerBumpsTheGeneration(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"LOG_LEVEL": "debug"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + genETag := func() string { |
| + t.Helper() |
| + _, etag, err := s.Get(ctx, GenKey("bab", "titogo")) |
| + if err != nil { |
| + t.Fatalf("read generation: %v", err) |
| + } |
| + return etag |
| + } |
| + before := genETag() |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", |
| + map[string]Var{"TENANT_SMTP_USER": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + afterWrite := genETag() |
| + if afterWrite == before { |
| + t.Fatal("writing the instance layer did not bump the generation, so the fast loop would never see it") |
| + } |
| + if err := DeleteInstanceConfig(ctx, s, "bab", "titogo", "book.customer.example.com"); err != nil { |
| + t.Fatalf("DeleteInstanceConfig: %v", err) |
| + } |
| + if genETag() == afterWrite { |
| + t.Fatal("removing the instance layer did not bump the generation, so the route would keep its last merge") |
| + } |
| + if _, _, found, err := LoadInstanceConfig(ctx, s, "bab", "titogo", "book.customer.example.com"); err != nil || found { |
| + t.Fatalf("the overlay survived its delete: found=%v err=%v", found, err) |
| + } |
| + // Deleting again is a no-op success, so a retried teardown converges. |
| + if err := DeleteInstanceConfig(ctx, s, "bab", "titogo", "book.customer.example.com"); err != nil { |
| + t.Fatalf("second DeleteInstanceConfig: %v", err) |
| + } |
| +} |
| + |
| +// The precedence the design states: default < environment < instance |
| +// (§6.2). Three layers, each winning per key over the one below it, and the |
| +// instance layer's etag handed back so the caller can later check what was |
| +// delivered. |
| +func TestLoadMergedHostResolvesDefaultThenEnvironmentThenInstance(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{ |
| + "FROM_DEFAULT": "default", "FROM_ENV": "default", "FROM_INSTANCE": "default"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + if _, err := ApplyEnvironment(ctx, s, "bab", "titogo", "prod", map[string]Var{ |
| + "FROM_ENV": {Kind: "env", Value: "environment"}, |
| + "FROM_INSTANCE": {Kind: "env", Value: "environment"}}, nil); err != nil { |
| + t.Fatalf("ApplyEnvironment: %v", err) |
| + } |
| + _, want, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", map[string]Var{ |
| + "FROM_INSTANCE": {Kind: "env", Value: "instance"}}, nil) |
| + if err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + |
| + m, etag, err := LoadMergedHost(ctx, s, "bab", "titogo", "prod", "book.customer.example.com") |
| + if err != nil { |
| + t.Fatalf("LoadMergedHost: %v", err) |
| + } |
| + if etag != want { |
| + t.Fatalf("etag = %q, want the overlay's own %q", etag, want) |
| + } |
| + for k, wantVal := range map[string]string{ |
| + "FROM_DEFAULT": "default", "FROM_ENV": "environment", "FROM_INSTANCE": "instance"} { |
| + if got := m.Vars[k].Value; got != wantVal { |
| + t.Errorf("%s = %q, want %q", k, got, wantVal) |
| + } |
| + } |
| + |
| + // A sibling with no overlay of its own gets the two-layer merge and an |
| + // empty etag, which is what says "this host has no overlay" rather than |
| + // "this agent did not render one". |
| + m2, etag2, err := LoadMergedHost(ctx, s, "bab", "titogo", "prod", "other.customer.example.com") |
| + if err != nil { |
| + t.Fatalf("LoadMergedHost (sibling): %v", err) |
| + } |
| + if etag2 != "" { |
| + t.Fatalf("a host with no overlay reported etag %q", etag2) |
| + } |
| + if m2.Vars["FROM_INSTANCE"].Value != "environment" { |
| + t.Fatalf("the sibling saw another instance's value: %+v", m2.Vars) |
| + } |
| +} |
| + |
| +// With no overlay, LoadMergedHost is LoadMerged, to the field. That matters |
| +// for the unbound case in particular: LoadMerged returns the default bundle |
| +// OBJECT there, exec_delivery and all, and a merge that quietly rebuilt it |
| +// would strip a flag other code reads off the same manifest. |
| +func TestLoadMergedHostWithNoOverlayIsExactlyLoadMerged(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"K": "v"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { |
| + t.Fatalf("SetExecDelivery: %v", err) |
| + } |
| + want, err := LoadMerged(ctx, s, "bab", "titogo", "") |
| + if err != nil { |
| + t.Fatalf("LoadMerged: %v", err) |
| + } |
| + got, etag, err := LoadMergedHost(ctx, s, "bab", "titogo", "", "book.customer.example.com") |
| + if err != nil || etag != "" { |
| + t.Fatalf("LoadMergedHost = %q %v", etag, err) |
| + } |
| + if got.ExecDelivery != want.ExecDelivery || got.Vars["K"] != want.Vars["K"] || len(got.Vars) != len(want.Vars) { |
| + t.Fatalf("LoadMergedHost diverged from LoadMerged with no overlay: %+v vs %+v", got, want) |
| + } |
| +} |
| + |
| +// exec_delivery is never read from this layer (the ruling at |
| +// appconfig.go:66-71). An overlay that carries the field does not put it in |
| +// the merged view, so nothing downstream can act on it. |
| +func TestInstanceLayerNeverCarriesExecDelivery(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"K": "v"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + putInstanceRaw(t, ctx, s, "bab", "titogo", "book.customer.example.com", &Manifest{ |
| + Schema: 1, App: "titogo", Vars: map[string]Var{"OWN": {Kind: "env", Value: "1"}}, |
| + ExecDelivery: ExecDeliveryFile}) |
| + m, _, err := LoadMergedHost(ctx, s, "bab", "titogo", "", "book.customer.example.com") |
| + if err != nil { |
| + t.Fatalf("LoadMergedHost: %v", err) |
| + } |
| + if m.ExecDelivery != "" { |
| + t.Fatalf("exec_delivery = %q: the instance layer must never decide delivery", m.ExecDelivery) |
| + } |
| + if m.Vars["OWN"].Value != "1" { |
| + t.Fatalf("the overlay's vars were dropped along with its flag: %+v", m.Vars) |
| + } |
| +} |
| + |
| +// A route bound to a bundle that does not exist is still an ERROR, overlay |
| +// or no overlay. The binding asserted the bundle exists; an overlay does not |
| +// make that assertion true, and answering with a partial merge would hand a |
| +// live route a file built from a binding nobody honoured. |
| +func TestLoadMergedHostStillRefusesAMissingBoundBundle(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"K": "v"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", |
| + map[string]Var{"OWN": {Kind: "env", Value: "1"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + if _, _, err := LoadMergedHost(ctx, s, "bab", "titogo", "gone", "book.customer.example.com"); err == nil { |
| + t.Fatal("a missing bound bundle resolved anyway because the host had an overlay") |
| + } |
| +} |
| + |
| +// The index the sweeps use: one listing per app answers "which of this |
| +// app's instances have an overlay at all", so a pass costs a Get only for |
| +// the instances that do. |
| +func TestInstanceOverlaysListsOnlyThisAppsHosts(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + have, err := instanceOverlays(ctx, s, "bab", "titogo") |
| + if err != nil || len(have) != 0 { |
| + t.Fatalf("empty app = %v %v, want an empty set and no error", have, err) |
| + } |
| + for _, host := range []string{"book.customer.example.com", "cart.customer.example.com"} { |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", host, |
| + map[string]Var{"OWN": {Kind: "env", Value: host}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance %s: %v", host, err) |
| + } |
| + } |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "eleven", "other.example.com", |
| + map[string]Var{"OWN": {Kind: "env", Value: "x"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance (other app): %v", err) |
| + } |
| + have, err = instanceOverlays(ctx, s, "bab", "titogo") |
| + if err != nil { |
| + t.Fatalf("instanceOverlays: %v", err) |
| + } |
| + if len(have) != 2 || !have["book.customer.example.com"] || !have["cart.customer.example.com"] { |
| + t.Fatalf("instanceOverlays = %v, want just this app's two hosts", have) |
| + } |
| + if have["other.example.com"] { |
| + t.Fatal("another app's overlay leaked into this app's index") |
| + } |
| +} |
| + |
| +// countingStore counts the reads and writes a pass makes, per key, so a |
| +// test can pin the COST of a feature and not just its output. The instance |
| +// layer's promise is one extra listing per app and one extra Get per |
| +// instance that actually has an overlay, which is a claim only a counter |
| +// can keep honest. |
| +type countingStore struct { |
| + release.Store |
| + mu sync.Mutex |
| + gets map[string]int |
| + lists map[string]int |
| + puts map[string]int |
| +} |
| + |
| +func newCountingStore(s release.Store) *countingStore { |
| + return &countingStore{Store: s, gets: map[string]int{}, lists: map[string]int{}, puts: map[string]int{}} |
| +} |
| + |
| +func (c *countingStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + c.mu.Lock() |
| + c.gets[key]++ |
| + c.mu.Unlock() |
| + return c.Store.Get(ctx, key) |
| +} |
| + |
| +func (c *countingStore) List(ctx context.Context, prefix string) ([]string, error) { |
| + c.mu.Lock() |
| + c.lists[prefix]++ |
| + c.mu.Unlock() |
| + return c.Store.List(ctx, prefix) |
| +} |
| + |
| +func (c *countingStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| + c.mu.Lock() |
| + c.puts[key]++ |
| + c.mu.Unlock() |
| + return c.Store.Put(ctx, key, body, ifMatch) |
| +} |
| + |
| +func (c *countingStore) n(m map[string]int, k string) int { |
| + c.mu.Lock() |
| + defer c.mu.Unlock() |
| + return m[k] |
| +} |
| + |
| +// total sums every count in m, so a test can assert "this pass wrote |
| +// nothing to the release store at all" without naming every key it might |
| +// have touched. |
| +func (c *countingStore) total(m map[string]int) int { |
| + c.mu.Lock() |
| + defer c.mu.Unlock() |
| + n := 0 |
| + for _, v := range m { |
| + n += v |
| + } |
| + return n |
| +} |
| + |
| +// syncInstanceKit is the unit-path fixture these tests share: a store, a |
| +// registry, an env dir and a parsed key pair. |
| +func syncInstanceKit(t *testing.T) (context.Context, release.Store, *registry.Registry, string, *ecdh.PrivateKey) { |
| + t.Helper() |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + reg := testRegistry(t) |
| + priv, _, err := GenerateKey() |
| + if err != nil { |
| + t.Fatalf("GenerateKey: %v", err) |
| + } |
| + privKey, err := ParsePrivateKey(priv, "test") |
| + if err != nil { |
| + t.Fatalf("ParsePrivateKey: %v", err) |
| + } |
| + return ctx, s, reg, t.TempDir(), privKey |
| +} |
| + |
| +func unitRoute(t *testing.T, reg *registry.Registry, host, app, acct, env string) { |
| + t.Helper() |
| + if err := reg.Upsert(registry.Route{ |
| + Host: host, Kind: "instance", App: app, Account: acct, Status: "ready", |
| + Environment: env, Backing: registry.BackingUnit, |
| + }); err != nil { |
| + t.Fatalf("Upsert %s: %v", host, err) |
| + } |
| +} |
| + |
| +// The delivery this layer exists for: one instance of an app gets a value |
| +// its siblings never see. The sibling's file is the app merge, untouched. |
| +func TestSyncRendersTheInstanceOverlayForItsOwnRouteOnly(t *testing.T) { |
| + ctx, s, reg, envDir, priv := syncInstanceKit(t) |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes", "TENANT": "none"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + unitRoute(t, reg, "book.example.com", "titogo", "bab", "") |
| + unitRoute(t, reg, "cart.example.com", "titogo", "bab", "") |
| + |
| + if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { |
| + t.Fatalf("Sync: %v", err) |
| + } |
| + book, err := os.ReadFile(filepath.Join(envDir, "book.example.com.env")) |
| + if err != nil { |
| + t.Fatalf("read book env: %v", err) |
| + } |
| + if !bytes.Contains(book, []byte("TENANT=book\n")) || !bytes.Contains(book, []byte("APP_WIDE=yes\n")) { |
| + t.Fatalf("the overlaid route's file is wrong:\n%s", book) |
| + } |
| + cart, err := os.ReadFile(filepath.Join(envDir, "cart.example.com.env")) |
| + if err != nil { |
| + t.Fatalf("read cart env: %v", err) |
| + } |
| + if bytes.Contains(cart, []byte("book")) || !bytes.Contains(cart, []byte("TENANT=none\n")) { |
| + t.Fatalf("a sibling saw another instance's config:\n%s", cart) |
| + } |
| + |
| + // The sweep and the single-shot API must agree about what a host gets. |
| + // The sweeps resolve through the group cache and never call |
| + // LoadMergedHost, so this is the only thing holding the two together. |
| + want, _, err := LoadMergedHost(ctx, s, "bab", "titogo", "", "book.example.com") |
| + if err != nil { |
| + t.Fatalf("LoadMergedHost: %v", err) |
| + } |
| + rendered, err := RenderEnv(want, priv) |
| + if err != nil { |
| + t.Fatalf("RenderEnv: %v", err) |
| + } |
| + if !bytes.Contains(book, rendered) { |
| + t.Fatalf("the pass and LoadMergedHost disagree:\npass:\n%s\nLoadMergedHost:\n%s", book, rendered) |
| + } |
| +} |
| + |
| +// The cost. One listing per app, a Get for the instance that has an |
| +// overlay, and NOTHING for the instance that does not. |
| +func TestSyncCostsOneListingPerAppAndOneGetPerOverlay(t *testing.T) { |
| + ctx, base, reg, envDir, priv := syncInstanceKit(t) |
| + if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + if _, _, err := ApplyInstance(ctx, base, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + unitRoute(t, reg, "book.example.com", "titogo", "bab", "") |
| + unitRoute(t, reg, "cart.example.com", "titogo", "bab", "") |
| + unitRoute(t, reg, "desk.example.com", "titogo", "bab", "") |
| + |
| + s := newCountingStore(base) |
| + if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { |
| + t.Fatalf("Sync: %v", err) |
| + } |
| + if got := s.n(s.lists, InstancesConfigPrefix("bab", "titogo")); got != 1 { |
| + t.Fatalf("instance overlay listings = %d, want exactly 1 for three routes of one app", got) |
| + } |
| + if got := s.n(s.gets, InstanceConfigKey("bab", "titogo", "book.example.com")); got != 1 { |
| + t.Fatalf("overlay Gets for book = %d, want 1", got) |
| + } |
| + for _, host := range []string{"cart.example.com", "desk.example.com"} { |
| + if got := s.n(s.gets, InstanceConfigKey("bab", "titogo", host)); got != 0 { |
| + t.Fatalf("overlay Gets for %s = %d, want none: the listing already said it has no overlay", host, got) |
| + } |
| + } |
| + // A convergence pass reads the release store, it never writes to it — |
| + // every write this test triggers (SetEnv, ApplyInstance) went through |
| + // the unwrapped base store during setup, before s ever saw a call. |
| + if got := s.total(s.puts); got != 0 { |
| + t.Fatalf("Sync put %d object(s) to the release store, want 0: this pass only writes local env files", got) |
| + } |
| +} |
| + |
| +// The unit pass never delivers an exec-backed route's config at all |
| +// (TestSyncSkipsExecBackedRoutes) — SyncExec does, and reports its own |
| +// overlay failures on it correctly. Final review, plan 4: the unit pass |
| +// used to fetch the overlay for EVERY syncable route regardless of Backing |
| +// and only checked Backing 53 lines later, so it paid a Get for exactly the |
| +// exec-backed, overlay-bearing routes step 5 targets, and threw the result |
| +// away. This pins the fixed cost: zero Gets against that key from the unit |
| +// pass, whatever the route's overlay actually holds. |
| +func TestSyncPaysNoGetForAnExecBackedRoutesOverlay(t *testing.T) { |
| + ctx, base, reg, envDir, priv := syncInstanceKit(t) |
| + if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + if _, _, err := ApplyInstance(ctx, base, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + // Exec-backed: Backing left at its zero value, the fleet default (same |
| + // shape as TestSyncSkipsExecBackedRoutes). |
| + if err := reg.Upsert(registry.Route{ |
| + Host: "book.example.com", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", |
| + }); err != nil { |
| + t.Fatalf("Upsert: %v", err) |
| + } |
| + s := newCountingStore(base) |
| + if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { |
| + t.Fatalf("Sync: %v", err) |
| + } |
| + if got := s.n(s.gets, InstanceConfigKey("bab", "titogo", "book.example.com")); got != 0 { |
| + t.Fatalf("unit pass Get'd an exec-backed route's overlay %d time(s), want 0: SyncExec owns delivering it, not Sync", got) |
| + } |
| + if _, err := os.Stat(filepath.Join(envDir, "book.example.com.env")); !os.IsNotExist(err) { |
| + t.Fatalf("expected no env file for an exec-backed route, stat err = %v", err) |
| + } |
| +} |
| + |
| +// A Get failure against an exec-backed route's overlay must be invisible to |
| +// the unit pass: gating the fetch on Backing == BackingUnit (rather than |
| +// merely discarding its result) means the failure never happens here at |
| +// all, so the pass cannot print the misleading "env file left as-is" claim |
| +// about a file it never writes for this route. SyncExec, exercised |
| +// separately, is the one that reports a failure on this exact key. |
| +func TestSyncPrintsNothingForAnExecBackedRoutesOverlayGetFailure(t *testing.T) { |
| + ctx, base, reg, envDir, priv := syncInstanceKit(t) |
| + if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + if _, _, err := ApplyInstance(ctx, base, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + if err := reg.Upsert(registry.Route{ |
| + Host: "book.example.com", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", |
| + }); err != nil { |
| + t.Fatalf("Upsert: %v", err) |
| + } |
| + failing := &failingKeyStore{Store: base, failKey: InstanceConfigKey("bab", "titogo", "book.example.com"), |
| + err: errors.New("s3: transient 503 SlowDown")} |
| + out := captureStdout(t, func() { |
| + if err := Sync(ctx, reg, failing, priv, envDir, func(string) error { return nil }, nil); err != nil { |
| + t.Fatalf("Sync: %v", err) |
| + } |
| + }) |
| + if strings.Contains(out, "unresolved") { |
| + t.Fatalf("the unit pass printed about an exec-backed route's overlay, which it never delivers or Gets:\n%s", out) |
| + } |
| + if !strings.Contains(out, "exec-backed") { |
| + t.Fatalf("expected the ordinary exec-backed notice (this route still has a default bundle), got:\n%s", out) |
| + } |
| + if _, err := os.Stat(filepath.Join(envDir, "book.example.com.env")); !os.IsNotExist(err) { |
| + t.Fatalf("expected no env file for an exec-backed route, stat err = %v", err) |
| + } |
| +} |
| + |
| +// Two independently corrupt overlays are two independent failures, not one. |
| +// The listing itself succeeds (both hosts are named), so the "once per app" |
| +// dedup for a failed LISTING must not swallow the second route's Get |
| +// failure — that would leave one of two stale credential files invisible, |
| +// which is exactly what this layer exists to surface. |
| +// |
| +// book converges with a valid overlay first, so the case below covers the |
| +// sharper claim too: an already-delivered route's file survives untouched |
| +// when its overlay later becomes unreadable, not just that a route that |
| +// never had a file stays fileless. |
| +func TestSyncReportsEachRoutesOverlayGetFailureSeparately(t *testing.T) { |
| + ctx, s, reg, envDir, priv := syncInstanceKit(t) |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + // Only book is on the box for the baseline pass, so cart genuinely never |
| + // has a file at any point — registering it before the baseline would |
| + // let it converge on the shared APP_WIDE bundle too, which would blur |
| + // the two cases this test is pinning apart. |
| + unitRoute(t, reg, "book.example.com", "titogo", "bab", "") |
| + |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { |
| + t.Fatalf("Sync (baseline): %v", err) |
| + } |
| + delivered, err := os.ReadFile(filepath.Join(envDir, "book.example.com.env")) |
| + if err != nil { |
| + t.Fatalf("read book's file after the baseline pass: %v", err) |
| + } |
| + if !bytes.Contains(delivered, []byte("TENANT=book\n")) { |
| + t.Fatalf("the baseline overlay did not land: %q", delivered) |
| + } |
| + unitRoute(t, reg, "cart.example.com", "titogo", "bab", "") |
| + |
| + // Schema 2 is refused by getIfExistsAtETag ("newer than this binary |
| + // understands"), which is a Get-time failure specific to the object |
| + // read, not a listing failure — instanceOverlays only reads keys, never |
| + // their bodies, so the listing itself succeeds for both hosts. Deleting |
| + // book's valid overlay first, then putting the corrupt one raw (Put with |
| + // ifMatch "" is create-only), is what turns an already-delivered route's |
| + // overlay unreadable, mid-flight. |
| + if err := DeleteInstanceConfig(ctx, s, "bab", "titogo", "book.example.com"); err != nil { |
| + t.Fatalf("DeleteInstanceConfig: %v", err) |
| + } |
| + putInstanceRaw(t, ctx, s, "bab", "titogo", "book.example.com", &Manifest{Schema: 2, App: "titogo"}) |
| + putInstanceRaw(t, ctx, s, "bab", "titogo", "cart.example.com", &Manifest{Schema: 2, App: "titogo"}) |
| + |
| + out := captureStdout(t, func() { |
| + if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { |
| + t.Fatalf("Sync: %v", err) |
| + } |
| + }) |
| + for _, host := range []string{"book.example.com", "cart.example.com"} { |
| + if !strings.Contains(out, host) { |
| + t.Fatalf("no line for %s's overlay failure, one route's stale file went unreported:\n%s", host, out) |
| + } |
| + } |
| + if n := strings.Count(out, "instance config unresolved"); n != 2 { |
| + t.Fatalf("instance config unresolved lines = %d, want 2 (one per corrupt overlay), got:\n%s", n, out) |
| + } |
| + // cart never had a file before this pass, and a Get failure must leave |
| + // that exactly as it is: no env file materializes from a resolved group |
| + // merge that an unresolved overlay was supposed to sit on top of. |
| + if _, err := os.Stat(filepath.Join(envDir, "cart.example.com.env")); !os.IsNotExist(err) { |
| + t.Fatalf("cart.example.com.env exists after an unresolved overlay: %v", err) |
| + } |
| + // book already had a delivered file, and its Get failure must leave that |
| + // file exactly as it was: an overlay going unreadable is not licence to |
| + // touch a route that already converged on it. |
| + cur, err := os.ReadFile(filepath.Join(envDir, "book.example.com.env")) |
| + if err != nil || !bytes.Equal(cur, delivered) { |
| + t.Fatalf("book's already-delivered file changed after its overlay became unreadable: %q, want %q (err %v)", cur, delivered, err) |
| + } |
| +} |
| + |
| +// A listing that fails is not a determination. The route keeps its file and |
| +// the pass says so, the same contract a bundle fetch failure already has. |
| +func TestSyncLeavesTheFileAloneWhenTheOverlayListingFails(t *testing.T) { |
| + ctx, base, reg, envDir, priv := syncInstanceKit(t) |
| + if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + unitRoute(t, reg, "book.example.com", "titogo", "bab", "") |
| + path := filepath.Join(envDir, "book.example.com.env") |
| + if err := os.WriteFile(path, []byte("APP_WIDE=stale\n"), 0o600); err != nil { |
| + t.Fatalf("stage file: %v", err) |
| + } |
| + failing := &failingListStore{Store: base, failPrefix: InstancesConfigPrefix("bab", "titogo"), |
| + err: errors.New("s3: transient 503 SlowDown")} |
| + out := captureStdout(t, func() { |
| + if err := Sync(ctx, reg, failing, priv, envDir, func(string) error { return nil }, nil); err != nil { |
| + t.Fatalf("Sync: %v", err) |
| + } |
| + }) |
| + cur, err := os.ReadFile(path) |
| + if err != nil || string(cur) != "APP_WIDE=stale\n" { |
| + t.Fatalf("the file moved on a listing failure: %q %v", cur, err) |
| + } |
| + if !strings.Contains(out, "book.example.com") || !strings.Contains(out, "left as-is") { |
| + t.Fatalf("the pass was silent about an unresolved instance layer:\n%s", out) |
| + } |
| +} |
| + |
| +// The dedup sentinel itself, pinned across TWO routes of one app sharing |
| +// one failed LISTING (review finding, plan 4 task 7 fix round 1): every |
| +// other test naming a listing failure uses a single route, so a mutation |
| +// that stopped telling a listing failure apart from a per-host one — |
| +// errors.As(...) always false, or instanceOverlayListErr's wrapper dropped |
| +// entirely — would still leave every one of them green. This asserts the |
| +// dedup's actual effect: one line, not two, with one host anchor naming |
| +// whichever route the pass happened to reach first. |
| +func TestSyncDedupsTheOverlayListingFailureAcrossTwoRoutesOfOneApp(t *testing.T) { |
| + ctx, base, reg, envDir, priv := syncInstanceKit(t) |
| + if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + unitRoute(t, reg, "book.example.com", "titogo", "bab", "") |
| + unitRoute(t, reg, "cart.example.com", "titogo", "bab", "") |
| + for _, host := range []string{"book.example.com", "cart.example.com"} { |
| + if err := os.WriteFile(filepath.Join(envDir, host+".env"), []byte("APP_WIDE=stale\n"), 0o600); err != nil { |
| + t.Fatalf("stage %s: %v", host, err) |
| + } |
| + } |
| + failing := &failingListStore{Store: base, failPrefix: InstancesConfigPrefix("bab", "titogo"), |
| + err: errors.New("s3: transient 503 SlowDown")} |
| + out := captureStdout(t, func() { |
| + if err := Sync(ctx, reg, failing, priv, envDir, func(string) error { return nil }, nil); err != nil { |
| + t.Fatalf("Sync: %v", err) |
| + } |
| + }) |
| + if n := strings.Count(out, "instance config unresolved"); n != 1 { |
| + t.Fatalf("instance config unresolved lines = %d, want exactly 1 for one shared LISTING failure across two routes, got:\n%s", n, out) |
| + } |
| + if n := strings.Count(out, "(seen at "); n != 1 { |
| + t.Fatalf("(seen at ...) host anchors = %d, want exactly 1, got:\n%s", n, out) |
| + } |
| + for _, host := range []string{"book.example.com", "cart.example.com"} { |
| + cur, err := os.ReadFile(filepath.Join(envDir, host+".env")) |
| + if err != nil || string(cur) != "APP_WIDE=stale\n" { |
| + t.Fatalf("%s moved on a listing failure: %q %v", host, cur, err) |
| + } |
| + } |
| +} |
| + |
| +// The exec mirror: SyncExec had NO listing-failure test at all before this |
| +// (its said-once-per-app contract, stated in the doc comment above the |
| +// overlayFor loop, was pinned nowhere). Same two-route shape as the Sync |
| +// test above, same three mutations it exists to catch. |
| +func TestSyncExecDedupsTheOverlayListingFailureAcrossTwoRoutesOfOneApp(t *testing.T) { |
| + ctx, base, reg, dd, privB64, _ := execKit(t) |
| + priv, err := ParsePrivateKey(privB64, "test") |
| + if err != nil { |
| + t.Fatalf("ParsePrivateKey: %v", err) |
| + } |
| + if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + for _, host := range []string{"book.example.com", "cart.example.com"} { |
| + if err := reg.Upsert(registry.Route{ |
| + Host: host, Kind: "instance", App: "titogo", Account: "bab", Status: "ready", |
| + Socket: "/run/carlos/" + host + ".sock", |
| + }); err != nil { |
| + t.Fatalf("Upsert %s: %v", host, err) |
| + } |
| + if err := os.WriteFile(filepath.Join(dd, host+".config.env"), []byte("OLD=1\n"), 0o600); err != nil { |
| + t.Fatalf("stage %s: %v", host, err) |
| + } |
| + } |
| + failing := &failingListStore{Store: base, failPrefix: InstancesConfigPrefix("bab", "titogo"), |
| + err: errors.New("s3: transient 503 SlowDown")} |
| + out := captureStdout(t, func() { |
| + if err := SyncExec(ctx, reg, failing, priv, dd, func(string) error { return nil }, "", nil); err != nil { |
| + t.Fatalf("SyncExec: %v", err) |
| + } |
| + }) |
| + if n := strings.Count(out, "instance config unresolved"); n != 1 { |
| + t.Fatalf("instance config unresolved lines = %d, want exactly 1 for one shared LISTING failure across two routes, got:\n%s", n, out) |
| + } |
| + if n := strings.Count(out, "(seen at "); n != 1 { |
| + t.Fatalf("(seen at ...) host anchors = %d, want exactly 1, got:\n%s", n, out) |
| + } |
| + for _, host := range []string{"book.example.com", "cart.example.com"} { |
| + cur, err := os.ReadFile(filepath.Join(dd, host+".config.env")) |
| + if err != nil || string(cur) != "OLD=1\n" { |
| + t.Fatalf("%s moved on a listing failure: %q %v", host, cur, err) |
| + } |
| + } |
| +} |
| + |
| +// failingListStore makes List fail for exactly one prefix, the listing |
| +// shape of failingKeyStore (syncexec_test.go). |
| +type failingListStore struct { |
| + release.Store |
| + failPrefix string |
| + err error |
| +} |
| + |
| +func (f *failingListStore) List(ctx context.Context, prefix string) ([]string, error) { |
| + if prefix == f.failPrefix { |
| + return nil, f.err |
| + } |
| + return f.Store.List(ctx, prefix) |
| +} |
| + |
| +// The CRITICAL 1 guard's second door. An app with no config.json, a route |
| +// bound to nothing, and an overlay: the merged view is that overlay ALONE, |
| +// which is a wholesale replacement of whatever a hand-staged file holds. |
| +// Refuse it, loudly, every pass, and name the way out. |
| +func TestSyncRefusesToDropHandStagedKeysForAnOverlayAlone(t *testing.T) { |
| + ctx, s, reg, envDir, priv := syncInstanceKit(t) |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + unitRoute(t, reg, "book.example.com", "titogo", "bab", "") |
| + path := filepath.Join(envDir, "book.example.com.env") |
| + staged := "HAND_STAGED=1\nTENANT=old\n" |
| + if err := os.WriteFile(path, []byte(staged), 0o600); err != nil { |
| + t.Fatalf("stage file: %v", err) |
| + } |
| + |
| + var restarted []string |
| + out := captureStdout(t, func() { |
| + if err := Sync(ctx, reg, s, priv, envDir, func(h string) error { |
| + restarted = append(restarted, h) |
| + return nil |
| + }, nil); err != nil { |
| + t.Fatalf("Sync: %v", err) |
| + } |
| + }) |
| + cur, err := os.ReadFile(path) |
| + if err != nil || string(cur) != staged { |
| + t.Fatalf("a hand-staged file was replaced by an overlay-alone render: %q %v", cur, err) |
| + } |
| + if len(restarted) != 0 { |
| + t.Fatalf("restarted %v for a route this pass refused", restarted) |
| + } |
| + if !strings.Contains(out, "HAND_STAGED") || !strings.Contains(out, "book.example.com") { |
| + t.Fatalf("the refusal does not name the route and the key it would have dropped:\n%s", out) |
| + } |
| +} |
| + |
| +// The same shape, self-healing: nothing is lost, so the route converges. |
| +// An app with no default bundle and an instance overlay is a legitimate |
| +// state (step 5 creates exactly that for a fresh instance), and this guard |
| +// must not freeze it forever. |
| +func TestSyncConvergesAnOverlayAloneWhenNothingIsLost(t *testing.T) { |
| + ctx, s, reg, envDir, priv := syncInstanceKit(t) |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + unitRoute(t, reg, "book.example.com", "titogo", "bab", "") |
| + if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { |
| + t.Fatalf("Sync: %v", err) |
| + } |
| + cur, err := os.ReadFile(filepath.Join(envDir, "book.example.com.env")) |
| + if err != nil { |
| + t.Fatalf("read env file: %v", err) |
| + } |
| + if !bytes.Contains(cur, []byte("TENANT=book\n")) { |
| + t.Fatalf("the overlay-alone render did not land:\n%s", cur) |
| + } |
| +} |
| + |
| +// And the case that must NOT change: no default bundle, no overlay, no |
| +// binding is still "never configured at all", and a never-configured app's |
| +// hand-staged file is not this pass's to touch. |
| +func TestSyncStillLeavesUnconfiguredAppsAlone(t *testing.T) { |
| + ctx, s, reg, envDir, priv := syncInstanceKit(t) |
| + unitRoute(t, reg, "book.example.com", "titogo", "bab", "") |
| + path := filepath.Join(envDir, "book.example.com.env") |
| + staged := "HAND_STAGED=1\n" |
| + if err := os.WriteFile(path, []byte(staged), 0o600); err != nil { |
| + t.Fatalf("stage file: %v", err) |
| + } |
| + if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { |
| + t.Fatalf("Sync: %v", err) |
| + } |
| + cur, err := os.ReadFile(path) |
| + if err != nil || string(cur) != staged { |
| + t.Fatalf("an unconfigured app's file moved: %q %v", cur, err) |
| + } |
| +} |
| + |
| +// The exec half of the same delivery: one activator-spawned instance gets |
| +// its own value, its sibling does not, and the listing is paid for once. |
| +func TestSyncExecRendersTheInstanceOverlayForItsOwnRouteOnly(t *testing.T) { |
| + ctx, base, reg, dd, privB64, _ := execKit(t) |
| + priv, err := ParsePrivateKey(privB64, "test") |
| + if err != nil { |
| + t.Fatalf("ParsePrivateKey: %v", err) |
| + } |
| + if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes", "TENANT": "none"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + if _, _, err := ApplyInstance(ctx, base, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + for _, host := range []string{"book.example.com", "cart.example.com"} { |
| + if err := reg.Upsert(registry.Route{ |
| + Host: host, Kind: "instance", App: "titogo", Account: "bab", Status: "ready", |
| + Socket: "/run/carlos/" + host + ".sock", |
| + }); err != nil { |
| + t.Fatalf("Upsert %s: %v", host, err) |
| + } |
| + } |
| + |
| + s := newCountingStore(base) |
| + if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", nil); err != nil { |
| + t.Fatalf("SyncExec: %v", err) |
| + } |
| + book, err := os.ReadFile(filepath.Join(dd, "book.example.com.config.env")) |
| + if err != nil { |
| + t.Fatalf("read book config: %v", err) |
| + } |
| + if !bytes.Contains(book, []byte("TENANT=book\n")) || !bytes.Contains(book, []byte("APP_WIDE=yes\n")) { |
| + t.Fatalf("the overlaid route's config file is wrong:\n%s", book) |
| + } |
| + cart, err := os.ReadFile(filepath.Join(dd, "cart.example.com.config.env")) |
| + if err != nil { |
| + t.Fatalf("read cart config: %v", err) |
| + } |
| + if bytes.Contains(cart, []byte("book")) || !bytes.Contains(cart, []byte("TENANT=none\n")) { |
| + t.Fatalf("a sibling saw another instance's config:\n%s", cart) |
| + } |
| + if got := s.n(s.lists, InstancesConfigPrefix("bab", "titogo")); got != 1 { |
| + t.Fatalf("instance overlay listings = %d, want exactly 1 for two routes of one app", got) |
| + } |
| + if got := s.n(s.gets, InstanceConfigKey("bab", "titogo", "cart.example.com")); got != 0 { |
| + t.Fatalf("read an overlay for a host the listing did not name (%d times)", got) |
| + } |
| +} |
| + |
| +// exec_delivery is a BUNDLE-level flag and the instance layer is not one of |
| +// the bundles it is read from (the ruling at appconfig.go:66-71). A TCP-addr |
| +// route whose app has never opted in gets no file, whatever its overlay |
| +// says — a per-host delivery flag is exactly the granularity that ruling |
| +// refused, and honouring one here would be the config-that-lies hazard it |
| +// names. |
| +func TestSyncExecNeverReadsExecDeliveryFromTheInstanceLayer(t *testing.T) { |
| + ctx, s, reg, dd, privB64, _ := execKit(t) |
| + priv, err := ParsePrivateKey(privB64, "test") |
| + if err != nil { |
| + t.Fatalf("ParsePrivateKey: %v", err) |
| + } |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + putInstanceRaw(t, ctx, s, "bab", "titogo", "router.example.com", &Manifest{ |
| + Schema: 1, App: "titogo", Vars: map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, |
| + ExecDelivery: ExecDeliveryFile}) |
| + if err := reg.Upsert(registry.Route{ |
| + Host: "router.example.com", Kind: "instance", App: "titogo", Account: "bab", |
| + Status: "ready", Addr: "127.0.0.1:8080", |
| + }); err != nil { |
| + t.Fatalf("Upsert: %v", err) |
| + } |
| + if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", nil); err != nil { |
| + t.Fatalf("SyncExec: %v", err) |
| + } |
| + if _, err := os.Stat(filepath.Join(dd, "router.example.com.config.env")); !os.IsNotExist(err) { |
| + t.Fatalf("an instance overlay opted a route into file delivery: %v", err) |
| + } |
| + |
| + // Opt the APP in, and the same route gets the same overlay merged in the |
| + // ordinary way — the flag's door is the bundle, the overlay's payload is |
| + // vars. |
| + if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { |
| + t.Fatalf("SetExecDelivery: %v", err) |
| + } |
| + if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", nil); err != nil { |
| + t.Fatalf("SyncExec (opted in): %v", err) |
| + } |
| + cur, err := os.ReadFile(filepath.Join(dd, "router.example.com.config.env")) |
| + if err != nil { |
| + t.Fatalf("read router config: %v", err) |
| + } |
| + if !bytes.Contains(cur, []byte("TENANT=book\n")) { |
| + t.Fatalf("the overlay's vars did not land once the app opted in:\n%s", cur) |
| + } |
| +} |
| + |
| +// The exec mirror of TestSyncReportsEachRoutesOverlayGetFailureSeparately: |
| +// two independently corrupt overlays are two independent failures, not one. |
| +// The listing itself succeeds (both hosts are named), so the "once per app" |
| +// dedup for a failed LISTING must not swallow the second route's Get |
| +// failure — that would leave one of two stale config files invisible, |
| +// which is exactly what this layer exists to surface. |
| +func TestSyncExecReportsEachRoutesOverlayGetFailureSeparately(t *testing.T) { |
| + ctx, s, reg, dd, privB64, _ := execKit(t) |
| + priv, err := ParsePrivateKey(privB64, "test") |
| + if err != nil { |
| + t.Fatalf("ParsePrivateKey: %v", err) |
| + } |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + // Schema 2 is refused by getIfExistsAtETag ("newer than this binary |
| + // understands"), which is a Get-time failure specific to the object |
| + // read, not a listing failure — instanceOverlays only reads keys, never |
| + // their bodies, so the listing itself succeeds for both hosts. |
| + putInstanceRaw(t, ctx, s, "bab", "titogo", "book.example.com", &Manifest{Schema: 2, App: "titogo"}) |
| + putInstanceRaw(t, ctx, s, "bab", "titogo", "cart.example.com", &Manifest{Schema: 2, App: "titogo"}) |
| + for _, host := range []string{"book.example.com", "cart.example.com"} { |
| + if err := reg.Upsert(registry.Route{ |
| + Host: host, Kind: "instance", App: "titogo", Account: "bab", Status: "ready", |
| + Socket: "/run/carlos/" + host + ".sock", |
| + }); err != nil { |
| + t.Fatalf("Upsert %s: %v", host, err) |
| + } |
| + } |
| + |
| + out := captureStdout(t, func() { |
| + if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", nil); err != nil { |
| + t.Fatalf("SyncExec: %v", err) |
| + } |
| + }) |
| + for _, host := range []string{"book.example.com", "cart.example.com"} { |
| + if !strings.Contains(out, host) { |
| + t.Fatalf("no line for %s's overlay failure, one route's stale file went unreported:\n%s", host, out) |
| + } |
| + } |
| + if n := strings.Count(out, "instance config unresolved"); n != 2 { |
| + t.Fatalf("instance config unresolved lines = %d, want 2 (one per corrupt overlay), got:\n%s", n, out) |
| + } |
| + // Neither route had a file before this pass, and a Get failure must |
| + // leave that exactly as it is: no config file materializes from a |
| + // resolved group merge that an unresolved overlay was supposed to sit |
| + // on top of. |
| + for _, host := range []string{"book.example.com", "cart.example.com"} { |
| + if _, err := os.Stat(filepath.Join(dd, host+".config.env")); !os.IsNotExist(err) { |
| + t.Fatalf("%s.config.env exists after an unresolved overlay: %v", host, err) |
| + } |
| + } |
| +} |
| + |
| +// An app with no default bundle at all, and one instance with an overlay. |
| +// The overlay is the whole configuration of that route, so its file is |
| +// written, not removed — while its sibling, which has nothing anywhere, is |
| +// reaped exactly as it is today. |
| +func TestSyncExecConvergesAnOverlayAloneInsteadOfRemoving(t *testing.T) { |
| + ctx, s, reg, dd, privB64, _ := execKit(t) |
| + priv, err := ParsePrivateKey(privB64, "test") |
| + if err != nil { |
| + t.Fatalf("ParsePrivateKey: %v", err) |
| + } |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + for _, host := range []string{"book.example.com", "cart.example.com"} { |
| + if err := reg.Upsert(registry.Route{ |
| + Host: host, Kind: "instance", App: "titogo", Account: "bab", Status: "ready", |
| + Socket: "/run/carlos/" + host + ".sock", |
| + }); err != nil { |
| + t.Fatalf("Upsert %s: %v", host, err) |
| + } |
| + if err := os.WriteFile(filepath.Join(dd, host+".config.env"), []byte("OLD=1\n"), 0o600); err != nil { |
| + t.Fatalf("stage %s: %v", host, err) |
| + } |
| + } |
| + if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", nil); err != nil { |
| + t.Fatalf("SyncExec: %v", err) |
| + } |
| + book, err := os.ReadFile(filepath.Join(dd, "book.example.com.config.env")) |
| + if err != nil { |
| + t.Fatalf("the overlaid route's file was removed: %v", err) |
| + } |
| + if !bytes.Contains(book, []byte("TENANT=book\n")) || bytes.Contains(book, []byte("OLD=1")) { |
| + t.Fatalf("the overlay-alone render did not converge:\n%s", book) |
| + } |
| + if _, err := os.Stat(filepath.Join(dd, "cart.example.com.config.env")); !os.IsNotExist(err) { |
| + t.Fatalf("a route with no config anywhere kept its file: %v", err) |
| + } |
| +} |
| + |
| +// The mass-removal guard, with an overlay in the middle of it. Two apps |
| +// whose manifests all read absent still freeze every removal on the box, |
| +// and the one route that has an overlay still converges: writing a file a |
| +// bundle asked for is not a removal, and the guard exists to stop |
| +// deletions, not delivery. |
| +func TestSyncExecMassRemovalGuardHoldsRemovalsButNotOverlays(t *testing.T) { |
| + ctx, s, reg, dd, privB64, _ := execKit(t) |
| + priv, err := ParsePrivateKey(privB64, "test") |
| + if err != nil { |
| + t.Fatalf("ParsePrivateKey: %v", err) |
| + } |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + routes := map[string]string{ |
| + "book.example.com": "titogo", |
| + "cart.example.com": "titogo", |
| + "desk.example.com": "eleven", |
| + } |
| + for host, app := range routes { |
| + if err := reg.Upsert(registry.Route{ |
| + Host: host, Kind: "instance", App: app, Account: "bab", Status: "ready", |
| + Socket: "/run/carlos/" + host + ".sock", |
| + }); err != nil { |
| + t.Fatalf("Upsert %s: %v", host, err) |
| + } |
| + if err := os.WriteFile(filepath.Join(dd, host+".config.env"), []byte("OLD=1\n"), 0o600); err != nil { |
| + t.Fatalf("stage %s: %v", host, err) |
| + } |
| + } |
| + out := captureStdout(t, func() { |
| + if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", nil); err != nil { |
| + t.Fatalf("SyncExec: %v", err) |
| + } |
| + }) |
| + if !strings.Contains(out, "refusing to remove any config files this pass") { |
| + t.Fatalf("the mass-removal guard did not fire:\n%s", out) |
| + } |
| + for _, host := range []string{"cart.example.com", "desk.example.com"} { |
| + cur, err := os.ReadFile(filepath.Join(dd, host+".config.env")) |
| + if err != nil || string(cur) != "OLD=1\n" { |
| + t.Fatalf("%s lost its file while the guard was up: %q %v", host, cur, err) |
| + } |
| + } |
| + book, err := os.ReadFile(filepath.Join(dd, "book.example.com.config.env")) |
| + if err != nil { |
| + t.Fatalf("the overlaid route was frozen by a guard about removals: %v", err) |
| + } |
| + if !bytes.Contains(book, []byte("TENANT=book\n")) { |
| + t.Fatalf("the overlay did not converge under the guard:\n%s", book) |
| + } |
| +} |
| + |
| +// The layer's only convergence signal is the generation object. The route |
| +// fingerprint cannot see a bucket write and must not start paying to try: |
| +// this pins both halves of that at once. |
| +func TestWritingAnOverlayMovesTheGenerationAndNotTheFingerprint(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + routes := []registry.Route{{ |
| + Host: "book.example.com", Kind: "instance", App: "titogo", Account: "bab", |
| + Status: "ready", Socket: "/run/carlos/book.sock", |
| + }} |
| + execBefore, unitBefore := ConfigRouteFingerprints(routes) |
| + _, genBefore, err := s.Get(ctx, GenKey("bab", "titogo")) |
| + if err != nil { |
| + t.Fatalf("read generation: %v", err) |
| + } |
| + |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + |
| + execAfter, unitAfter := ConfigRouteFingerprints(routes) |
| + if execAfter != execBefore || unitAfter != unitBefore { |
| + t.Fatal("the route fingerprint moved for a bucket-side write, which would cost a read per route per tick") |
| + } |
| + _, genAfter, err := s.Get(ctx, GenKey("bab", "titogo")) |
| + if err != nil { |
| + t.Fatalf("read generation: %v", err) |
| + } |
| + if genAfter == genBefore { |
| + t.Fatal("the generation did not move, so the fast loop has nothing to notice and delivery waits out the hourly floor") |
| + } |
| +} |
| + |
| +// One pass over a box holding every shape this layer comes in: an instance |
| +// with an overlay over its app's bundle, a sibling with none, and an |
| +// instance whose app has no bundle at all so its overlay stands alone. |
| +// Then the overlay is removed and the route converges back, with the ack |
| +// saying so. |
| +func TestOnePassOverEveryInstanceLayerShape(t *testing.T) { |
| + ctx, s, reg, dd, privB64, _ := execKit(t) |
| + priv, err := ParsePrivateKey(privB64, "test") |
| + if err != nil { |
| + t.Fatalf("ParsePrivateKey: %v", err) |
| + } |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes", "TENANT": "none"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + _, bookETag, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil) |
| + if err != nil { |
| + t.Fatalf("ApplyInstance (book): %v", err) |
| + } |
| + _, deskETag, err := ApplyInstance(ctx, s, "bab", "solo", "desk.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "desk"}}, nil) |
| + if err != nil { |
| + t.Fatalf("ApplyInstance (desk): %v", err) |
| + } |
| + for host, app := range map[string]string{ |
| + "book.example.com": "titogo", "cart.example.com": "titogo", "desk.example.com": "solo"} { |
| + if err := reg.Upsert(registry.Route{ |
| + Host: host, Kind: "instance", App: app, Account: "bab", Status: "ready", |
| + Socket: "/run/carlos/" + host + ".sock", |
| + }); err != nil { |
| + t.Fatalf("Upsert %s: %v", host, err) |
| + } |
| + } |
| + |
| + acks := map[string]ConfigAck{} |
| + collect := func(a ConfigAck) { acks[a.Host] = a } |
| + if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", collect); err != nil { |
| + t.Fatalf("SyncExec: %v", err) |
| + } |
| + read := func(host string) string { |
| + t.Helper() |
| + body, err := os.ReadFile(filepath.Join(dd, host+".config.env")) |
| + if err != nil { |
| + t.Fatalf("read %s: %v", host, err) |
| + } |
| + return string(body) |
| + } |
| + if got := read("book.example.com"); !strings.Contains(got, "TENANT=book\n") || !strings.Contains(got, "APP_WIDE=yes\n") { |
| + t.Fatalf("book: %q", got) |
| + } |
| + if got := read("cart.example.com"); !strings.Contains(got, "TENANT=none\n") { |
| + t.Fatalf("cart saw another instance's config: %q", got) |
| + } |
| + if got := read("desk.example.com"); got != "TENANT=desk\n" { |
| + t.Fatalf("desk, whose app has no bundle, should render its overlay alone: %q", got) |
| + } |
| + for host, want := range map[string]string{ |
| + "book.example.com": bookETag, "cart.example.com": "", "desk.example.com": deskETag} { |
| + a, ok := acks[host] |
| + if !ok { |
| + t.Fatalf("no ack for %s", host) |
| + } |
| + if a.Schema != ConfigAckSchema || a.InstanceConfigETag != want { |
| + t.Fatalf("%s ack = %+v, want schema %d and etag %q", host, a, ConfigAckSchema, want) |
| + } |
| + if v := InstanceConfigDelivery(&a, want); v != InstanceConfigDelivered { |
| + t.Fatalf("%s delivery = %q, want %q", host, v, InstanceConfigDelivered) |
| + } |
| + } |
| + |
| + // Remove one overlay: that route converges back to its app's merge, and |
| + // its ack stops naming an overlay, which is what tells a reader the |
| + // removal actually reached the box. |
| + if err := DeleteInstanceConfig(ctx, s, "bab", "titogo", "book.example.com"); err != nil { |
| + t.Fatalf("DeleteInstanceConfig: %v", err) |
| + } |
| + acks = map[string]ConfigAck{} |
| + if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", collect); err != nil { |
| + t.Fatalf("SyncExec (after delete): %v", err) |
| + } |
| + if got := read("book.example.com"); !strings.Contains(got, "TENANT=none\n") { |
| + t.Fatalf("book did not converge back to its app's merge: %q", got) |
| + } |
| + if a := acks["book.example.com"]; a.InstanceConfigETag != "" || a.Schema != ConfigAckSchema { |
| + t.Fatalf("book ack after the delete = %+v, want the schema and no overlay", a) |
| + } |
| + if got := read("desk.example.com"); got != "TENANT=desk\n" { |
| + t.Fatalf("desk was disturbed by a sibling app's delete: %q", got) |
| + } |
| +} |
| + |
| +// The unit-path half of the removal claim. TestOnePassOverEveryInstanceLayerShape |
| +// pins the exec sweep's removal convergence; nothing else in the package |
| +// runs Sync (root's unit path) after a delete, so without this the claim |
| +// that a route "goes back to what the bundles alone say" once its overlay |
| +// is removed is only ever true for SyncExec. |
| +func TestSyncConvergesBackToTheBundleAfterAnOverlayIsDeleted(t *testing.T) { |
| + ctx, s, reg, envDir, priv := syncInstanceKit(t) |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes", "TENANT": "none"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + unitRoute(t, reg, "book.example.com", "titogo", "bab", "") |
| + |
| + if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { |
| + t.Fatalf("Sync: %v", err) |
| + } |
| + before, err := os.ReadFile(filepath.Join(envDir, "book.example.com.env")) |
| + if err != nil { |
| + t.Fatalf("read env file before the delete: %v", err) |
| + } |
| + if !bytes.Contains(before, []byte("TENANT=book\n")) { |
| + t.Fatalf("the overlay did not land before the delete: %q", before) |
| + } |
| + |
| + if err := DeleteInstanceConfig(ctx, s, "bab", "titogo", "book.example.com"); err != nil { |
| + t.Fatalf("DeleteInstanceConfig: %v", err) |
| + } |
| + if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { |
| + t.Fatalf("Sync (after delete): %v", err) |
| + } |
| + after, err := os.ReadFile(filepath.Join(envDir, "book.example.com.env")) |
| + if err != nil { |
| + t.Fatalf("read env file after the delete: %v", err) |
| + } |
| + if bytes.Contains(after, []byte("TENANT=book\n")) || !bytes.Contains(after, []byte("TENANT=none\n")) { |
| + t.Fatalf("the route did not converge back to the bundle alone after the overlay was deleted: %q", after) |
| + } |
| +} |
| + |
| +// Every other test in this file passes a no-op restart callback, so a |
| +// regression that made resolving one route's overlay spuriously mark a |
| +// different route as changed would still pass the whole suite. This one |
| +// records which hosts restart is actually called for. The first pass |
| +// converges both routes from nothing and restarts both once, which is |
| +// ordinary first-convergence behaviour and not the claim under test; that |
| +// recording is discarded before the pass that matters, where only book's |
| +// overlay changes and cart's rendered bytes do not move at all. |
| +func TestSyncDoesNotRestartAnUnchangedSiblingWhileAnOverlayIsResolved(t *testing.T) { |
| + ctx, s, reg, envDir, priv := syncInstanceKit(t) |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes", "TENANT": "none"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + unitRoute(t, reg, "book.example.com", "titogo", "bab", "") |
| + unitRoute(t, reg, "cart.example.com", "titogo", "bab", "") |
| + |
| + var restarted []string |
| + record := func(host string) error { |
| + restarted = append(restarted, host) |
| + return nil |
| + } |
| + if err := Sync(ctx, reg, s, priv, envDir, record, nil); err != nil { |
| + t.Fatalf("Sync (baseline): %v", err) |
| + } |
| + restarted = nil |
| + |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + if err := Sync(ctx, reg, s, priv, envDir, record, nil); err != nil { |
| + t.Fatalf("Sync (overlay pass): %v", err) |
| + } |
| + if len(restarted) != 1 || restarted[0] != "book.example.com" { |
| + t.Fatalf("restarted = %v, want exactly [book.example.com]: cart has no overlay and its rendered file did not change, so it must not restart", restarted) |
| + } |
| +} |
| + |
| +// The exec mirror of the test above, over bounce rather than restart. |
| +func TestSyncExecDoesNotBounceAnUnchangedSiblingWhileAnOverlayIsResolved(t *testing.T) { |
| + ctx, s, reg, dd, privB64, _ := execKit(t) |
| + priv, err := ParsePrivateKey(privB64, "test") |
| + if err != nil { |
| + t.Fatalf("ParsePrivateKey: %v", err) |
| + } |
| + if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes", "TENANT": "none"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + for _, host := range []string{"book.example.com", "cart.example.com"} { |
| + if err := reg.Upsert(registry.Route{ |
| + Host: host, Kind: "instance", App: "titogo", Account: "bab", Status: "ready", |
| + Socket: "/run/carlos/" + host + ".sock", |
| + }); err != nil { |
| + t.Fatalf("Upsert %s: %v", host, err) |
| + } |
| + } |
| + |
| + var bounced []string |
| + record := func(host string) error { |
| + bounced = append(bounced, host) |
| + return nil |
| + } |
| + if err := SyncExec(ctx, reg, s, priv, dd, record, "", nil); err != nil { |
| + t.Fatalf("SyncExec (baseline): %v", err) |
| + } |
| + bounced = nil |
| + |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + if err := SyncExec(ctx, reg, s, priv, dd, record, "", nil); err != nil { |
| + t.Fatalf("SyncExec (overlay pass): %v", err) |
| + } |
| + if len(bounced) != 1 || bounced[0] != "book.example.com" { |
| + t.Fatalf("bounced = %v, want exactly [book.example.com]: cart has no overlay and its rendered file did not change, so it must not bounce", bounced) |
| + } |
| +} |
diff --git a/internal/appconfig/routefingerprint.go b/internal/appconfig/routefingerprint.go| index 10cd89e27..45550090b 100644 |
| --- a/internal/appconfig/routefingerprint.go |
| +++ b/internal/appconfig/routefingerprint.go |
| @@ -51,6 +51,14 @@ import ( |
| // app's generation object (SetExecDeliveryAt), which the loop already |
| // watches. |
| // |
| +// The INSTANCE CONFIG LAYER is deliberately not here either, and for the |
| +// same reason with a sharper edge: apps/<acct>/<app>/instances-config/ |
| +// <host>.json is a bucket object, so seeing a change to one would cost a |
| +// listing per app per tick at best and a read per route at worst. It does |
| +// not need to be seen here — writing or deleting an overlay bumps the app's |
| +// generation object (ApplyInstance, DeleteInstanceConfig), which this loop |
| +// already watches, and that is the layer's whole signal. |
| +// |
| // Two digests, not one, because the two halves are delivered by different |
| // processes: the exec half by the agent itself (SyncExec), the unit half |
| // by a root pass the agent can only REQUEST. A shared digest would ask for |
diff --git a/internal/appconfig/store.go b/internal/appconfig/store.go| index de97f2217..b78f5597d 100644 |
| --- a/internal/appconfig/store.go |
| +++ b/internal/appconfig/store.go |
| @@ -48,28 +48,39 @@ func getIfExists(ctx context.Context, s release.Store, acct, app string) (*Manif |
| // getIfExistsAt is getIfExists generalized to any manifest key — Key(acct, |
| // app) for the default bundle, EnvironmentKey(acct, app, name) for a named |
| -// one (see environments.go). Same shape either way: a config.json and a |
| -// named environment bundle are the identical manifest format at sibling |
| -// bucket paths, so one read path serves both. |
| +// one (see environments.go), InstanceConfigKey(acct, app, host) for an |
| +// instance overlay (instances.go). Same shape either way: all three are the |
| +// identical manifest format at sibling bucket paths, so one read path |
| +// serves them. |
| func getIfExistsAt(ctx context.Context, s release.Store, key string) (*Manifest, bool, error) { |
| - raw, _, err := s.Get(ctx, key) |
| + m, _, found, err := getIfExistsAtETag(ctx, s, key) |
| + return m, found, err |
| +} |
| + |
| +// getIfExistsAtETag is getIfExistsAt plus the object's etag, which the |
| +// instance layer's reader needs and the other two bundles' readers do not: |
| +// the delivery ack names the etag of the overlay the agent rendered |
| +// (senders design §6.5), and an etag dropped here could not be recovered |
| +// later without a second read racing the first. |
| +func getIfExistsAtETag(ctx context.Context, s release.Store, key string) (*Manifest, string, bool, error) { |
| + raw, etag, err := s.Get(ctx, key) |
| if err != nil { |
| if errors.Is(err, os.ErrNotExist) { |
| - return nil, false, nil |
| + return nil, "", false, nil |
| } |
| - return nil, false, err |
| + return nil, "", false, err |
| } |
| var m Manifest |
| if err := json.Unmarshal(raw, &m); err != nil { |
| - return nil, false, fmt.Errorf("config %s: %w", key, err) |
| + return nil, "", false, fmt.Errorf("config %s: %w", key, err) |
| } |
| // json.Unmarshal has already dropped any fields this binary doesn't |
| // know about, so acting on a newer-schema manifest would silently |
| // discard whatever those fields meant. Refuse instead. |
| if m.Schema > 1 { |
| - return nil, false, fmt.Errorf("config %s: schema %d is newer than this binary understands — upgrade carlos", key, m.Schema) |
| + return nil, "", false, fmt.Errorf("config %s: schema %d is newer than this binary understands — upgrade carlos", key, m.Schema) |
| } |
| - return &m, true, nil |
| + return &m, etag, true, nil |
| } |
| // getForWriteAt is Get plus the two bits a caller needs to write back: the |
| @@ -119,7 +130,7 @@ func apply(ctx context.Context, s release.Store, acct, app, bundle string, sets |
| if !release.ValidApp(app) { |
| return 0, fmt.Errorf("config: invalid app %q", app) |
| } |
| - removed, err := applyAt(ctx, s, Key(acct, app), app, bundle, sets, unsets) |
| + removed, _, err := applyAt(ctx, s, Key(acct, app), app, bundle, sets, unsets) |
| if err != nil { |
| return 0, err |
| } |
| @@ -143,10 +154,10 @@ func apply(ctx context.Context, s release.Store, acct, app, bundle string, sets |
| // environment bundles that then satisfied `carlos route -environment`'s |
| // existence rail. Sets still create, which is how a bundle is meant to come |
| // into being. |
| -func applyAt(ctx context.Context, s release.Store, key, app, bundle string, sets map[string]Var, unsets []string) (int, error) { |
| +func applyAt(ctx context.Context, s release.Store, key, app, bundle string, sets map[string]Var, unsets []string) (int, string, error) { |
| for k, v := range sets { |
| if !ValidVarName(k) { |
| - return 0, fmt.Errorf("config: invalid var name %q", k) |
| + return 0, "", fmt.Errorf("config: invalid var name %q", k) |
| } |
| // Env values are validated here, while the operator can still see |
| // the plaintext — a bad value that only failed at render time would |
| @@ -155,16 +166,16 @@ func applyAt(ctx context.Context, s release.Store, key, app, bundle string, sets |
| // CLI, before sealing; by this point they're opaque envelopes. |
| if v.Kind == "env" { |
| if err := ValidVarValue(v.Value); err != nil { |
| - return 0, fmt.Errorf("config: var %s: %w", k, err) |
| + return 0, "", fmt.Errorf("config: var %s: %w", k, err) |
| } |
| } |
| } |
| m, etag, found, err := getForWriteAt(ctx, s, key, app) |
| if err != nil { |
| - return 0, err |
| + return 0, "", err |
| } |
| if !found && len(sets) == 0 { |
| - return 0, fmt.Errorf("config: %s has no %s bundle — nothing to unset", app, bundle) |
| + return 0, "", fmt.Errorf("config: %s has no %s bundle — nothing to unset", app, bundle) |
| } |
| if m.Vars == nil { |
| m.Vars = map[string]Var{} |
| @@ -182,12 +193,16 @@ func applyAt(ctx context.Context, s release.Store, key, app, bundle string, sets |
| m.Schema, m.App = 1, app |
| raw, err := json.MarshalIndent(m, "", " ") |
| if err != nil { |
| - return 0, err |
| + return 0, "", err |
| } |
| - if _, err := s.Put(ctx, key, raw, etag); err != nil { |
| - return 0, err |
| + // The new etag is returned because the instance layer's writer (design |
| + // §6.5) has to hand it to whoever will later compare it with an agent's |
| + // delivery ack. The other two bundles' callers ignore it. |
| + newETag, err := s.Put(ctx, key, raw, etag) |
| + if err != nil { |
| + return 0, "", err |
| } |
| - return removed, nil |
| + return removed, newETag, nil |
| } |
| // DefaultBundle is how the default bundle (config.json) is named to a human |
diff --git a/internal/appconfig/sync.go b/internal/appconfig/sync.go| index 0fe6d4d6c..5e7135fdc 100644 |
| --- a/internal/appconfig/sync.go |
| +++ b/internal/appconfig/sync.go |
| @@ -12,6 +12,7 @@ import ( |
| "bytes" |
| "context" |
| "crypto/ecdh" |
| + "errors" |
| "fmt" |
| "os" |
| "path/filepath" |
| @@ -67,6 +68,9 @@ func Sync(ctx context.Context, reg *registry.Registry, s release.Store, priv *ec |
| // render is not an overlay, it is a wholesale replacement, so |
| // syncHost refuses to let it drop keys the file on disk already holds |
| // (see its doc comment; 2026-08-02 review, CRITICAL 1 and round 2). |
| + // The materialization loop sets the same flag for an UNBOUND route |
| + // whose only config is its instance overlay (senders design §6.2), |
| + // which is the same replacement reached by a different door. |
| defaultMissing bool |
| } |
| resolved := map[groupKey]resolution{} |
| @@ -140,11 +144,27 @@ func Sync(ctx context.Context, reg *registry.Registry, s release.Store, priv *ec |
| } |
| resolved[k] = resolution{m: m, defaultMissing: !def.found} |
| } |
| + // The instance layer (senders design §6.2), resolved per route AFTER the |
| + // group base above. The group is still (Account, App, Environment) and |
| + // the default bundle is still read once per app: only the third layer is |
| + // per host, and it costs one listing per app plus a Get for each |
| + // instance that actually has an overlay. |
| + overlayFor := instanceOverlayLoader(ctx, s) |
| + // One line per app when the LISTING fails, not one per route: every |
| + // route of that app fails identically and for the same reason, since |
| + // instanceOverlayLoader caches that failure per (account, app) — |
| + // independent of Environment, which is why this is keyed on appKey and |
| + // not groupKey. A per-host Get or parse failure is a DIFFERENT claim — |
| + // two routes can have two separately corrupt overlay objects, each |
| + // wrong for its own reason — so that case is never folded in here; see |
| + // the errors.As branch below. |
| + said := map[appKey]bool{} |
| for _, rt := range routes { |
| if !syncable(rt) { |
| continue |
| } |
| - r := resolved[groupKey{release.RouteAccount(rt), rt.App, rt.Environment}] |
| + k := groupKey{release.RouteAccount(rt), rt.App, rt.Environment} |
| + r := resolved[k] |
| if r.err != nil { |
| // Loud, per route, every pass: this route is running on a stale |
| // env file until an operator fixes it. Its file is left exactly |
| @@ -156,10 +176,75 @@ func Sync(ctx context.Context, reg *registry.Registry, s release.Store, priv *ec |
| fmt.Printf(" %s: config unresolved (environment %q), env file left as-is: %v\n", rt.Host, env, r.err) |
| continue |
| } |
| + // The instance overlay wins per key over the group's merge, and is |
| + // fetched BEFORE the m == nil skip below: an unbound route of an app |
| + // with no default bundle used to resolve to "never configured" and |
| + // stop here, but with an overlay it now resolves to something (senders |
| + // design §6.2), so the skip has to see the overlay first. |
| + // |
| + // Fetched ONLY for unit-backed routes (final review, plan 4): an |
| + // exec-backed route never reaches syncHost below — SyncExec renders |
| + // its overlay and reports its own listing/Get failures on it — so a |
| + // Get paid here would be charged and discarded for every exec-backed |
| + // route that has one, which is exactly the shape step 5 targets. A |
| + // Get or parse failure paid here for such a route would also print |
| + // "env file left as-is" about a file this pass never writes, a claim |
| + // SyncExec already makes correctly on its own. Deliberately NOT |
| + // hoisting the rt.Backing check up to gate this whole block, though: |
| + // that would make the "exec-backed" print below fire for an |
| + // exec-backed route with no config at all, which it does not do |
| + // today — the check stays exactly where it always was, and only the |
| + // overlay fetch itself is narrowed. |
| + var inst *Manifest |
| + var instETag string |
| + if rt.Backing == registry.BackingUnit { |
| + var err error |
| + inst, instETag, err = overlayFor(k.acct, k.app, rt.Host) |
| + if err != nil { |
| + var listErr *instanceOverlayListErr |
| + if errors.As(err, &listErr) { |
| + // Said ONCE per (account, app), not once per route: the |
| + // listing this failed on is cached per app inside the |
| + // loader, so every route of the app, across every |
| + // environment it is bound to, fails identically and for |
| + // the same reason; a line each would bury the cause under |
| + // its own repetitions on a box with many routes. |
| + ak := appKey{k.acct, k.app} |
| + if !said[ak] { |
| + said[ak] = true |
| + // rt.Host anchors the line to a real route (the |
| + // first one this pass happened to reach), so an |
| + // operator scanning the log has something to grep |
| + // for; every other route of this app is silently |
| + // covered by the same line. |
| + fmt.Printf(" %s/%s: instance config unresolved (seen at %s), env files left as-is: %v\n", k.acct, k.app, rt.Host, err) |
| + } |
| + } else { |
| + // A per-host Get or parse failure, unlike the listing |
| + // above, is specific to THIS route: the listing already |
| + // succeeded (it named this host), so another route of |
| + // the same app can have a perfectly good overlay, or a |
| + // failure of its own for a different reason. Folding this |
| + // into the app-level line would leave that route's stale |
| + // credential file invisible, so it is reported every time. |
| + fmt.Printf(" %s: instance config unresolved, env file left as-is: %v\n", rt.Host, err) |
| + } |
| + continue |
| + } |
| + } |
| m := r.m |
| + if m == nil && inst == nil { |
| + continue // never configured at all: this file is not ours to touch |
| + } |
| + // An overlay with no group base behind it is a wholesale replacement, |
| + // not an overlay. it is the same state a bound route with no default |
| + // bundle is in, reached without a binding. It gets the same guard: |
| + // see syncHost. |
| + defaultMissing := r.defaultMissing |
| if m == nil { |
| - continue |
| + defaultMissing = true |
| } |
| + m = applyInstanceOverlay(k.app, m, inst) |
| // Only unit-backed routes read /etc/carlos/apps — an exec-backed |
| // route's env comes from <DataDir>/<host>.env, which the activator |
| // owns (it also carries CARLOS_ADMIN_TOKEN, so it can't be blindly |
| @@ -172,7 +257,7 @@ func Sync(ctx context.Context, reg *registry.Registry, s release.Store, priv *ec |
| fmt.Printf(" %s: exec-backed — config delivered by the agent's in-process loop to /data/<host>.config.env, not this pass\n", rt.Host) |
| continue |
| } |
| - syncHost(rt, m, r.defaultMissing, priv, envDir, restart, ack) |
| + syncHost(rt, m, defaultMissing, priv, envDir, restart, instETag, ack) |
| } |
| // An empty route list is never authority to delete anything: |
| // registry.Open silently creates an empty database when handed a wrong |
| @@ -249,7 +334,10 @@ func reap(routes []registry.Route, envDir string) { |
| // render-diff-write-restart path at all). RenderEnv emits exactly one |
| // KEY=value line per var, so the two key sets are the same by construction — |
| // except for the agent-owned names applyAgentVars adds to and removes from |
| -// the rendered file afterward, which is why they are skipped below. |
| +// the rendered file afterward, which is why they are skipped below. m is the |
| +// route's FULL merge, all three layers of it (default, environment, |
| +// instance), because the question is what the file would lose, not which |
| +// bundle a key came from. |
| // |
| // Blank lines and #-comments are ignored, matching how systemd's |
| // EnvironmentFile reads these files: they carry no configuration, so their |
| @@ -336,11 +424,15 @@ func UnitConfigApps(routes []registry.Route) [][2]string { |
| // defaultMissing says the merged view is a bound environment's bundle ALONE |
| // because the app has no config.json — see the guard below. |
| // |
| +// instETag is the etag of the instance overlay this route's merge already |
| +// folded in (empty when the route has no overlay), carried through to the |
| +// ack unchanged — see ConfigAck.InstanceConfigETag. |
| +// |
| // ack, if non-nil, is called once this route's file has actually changed |
| // and a restart was attempted — never on the "up to date" early return, |
| // since nothing was delivered on that pass and the ack already on file (if |
| // any) from a previous pass still describes what the route is running. |
| -func syncHost(rt registry.Route, m *Manifest, defaultMissing bool, priv *ecdh.PrivateKey, envDir string, restart func(host string) error, ack func(ConfigAck)) { |
| +func syncHost(rt registry.Route, m *Manifest, defaultMissing bool, priv *ecdh.PrivateKey, envDir string, restart func(host string) error, instETag string, ack func(ConfigAck)) { |
| // The host is joined into a root-written path; a registry row carrying |
| // a separator must never place a file outside envDir. Same trust level |
| // as release's live/<host> symlinks, but this one is one line to pin. |
| @@ -387,21 +479,29 @@ func syncHost(rt registry.Route, m *Manifest, defaultMissing bool, priv *ecdh.Pr |
| // properly, at the cost of changing the rendered file format. |
| if defaultMissing && existed && len(cur) > 0 { |
| lost, parsed := lostKeys(cur, m) |
| + // Which overlay is standing alone. A bound route names its |
| + // environment; an unbound one is here because this instance has its |
| + // own config layer (senders design §6.2) and its app has no default |
| + // bundle, and naming an empty environment would be a riddle. |
| + alone := "this instance's own config layer" |
| + if rt.Environment != "" { |
| + alone = fmt.Sprintf("environment %q", rt.Environment) |
| + } |
| switch { |
| case !parsed: |
| // Unparseable content has unknown keys, so nothing here can prove |
| // the render drops none of them. Assume the worst — this guard's |
| // whole job is to be wrong in the safe direction. |
| - fmt.Printf(" %s: environment %q is bound but %s has no default config bundle, so the merged view is that environment ALONE — "+ |
| + fmt.Printf(" %s: %s has no default config bundle, so the merged view is %s ALONE — "+ |
| "refusing to replace %s, which this pass cannot parse as KEY=value and so cannot prove is safe to overwrite; "+ |
| "import its config (`carlos env set --app %s KEY=value ...`) or remove the file\n", |
| - rt.Host, rt.Environment, rt.App, path, rt.App) |
| + rt.Host, rt.App, alone, path, rt.App) |
| return |
| case len(lost) > 0: |
| - fmt.Printf(" %s: environment %q is bound but %s has no default config bundle, so the merged view is that environment ALONE — "+ |
| + fmt.Printf(" %s: %s has no default config bundle, so the merged view is %s ALONE — "+ |
| "refusing to drop %d var(s) already in %s (%s); import them (`carlos env set --app %s KEY=value ...`) "+ |
| "and this route converges on the next pass, or remove the file if it is dead\n", |
| - rt.Host, rt.Environment, rt.App, len(lost), path, strings.Join(lost, ", "), rt.App) |
| + rt.Host, rt.App, alone, len(lost), path, strings.Join(lost, ", "), rt.App) |
| return |
| } |
| } |
| @@ -448,7 +548,7 @@ func syncHost(rt registry.Route, m *Manifest, defaultMissing bool, priv *ecdh.Pr |
| // route's shape, the relay identities (if any) it just wrote, and |
| // whether the restart below woke a real process or found a hibernating |
| // one with nothing to wake. |
| - a := ConfigAck{Host: rt.Host, Shape: shapeOf(rt), At: time.Now()} |
| + a := ConfigAck{Schema: ConfigAckSchema, Host: rt.Host, Shape: shapeOf(rt), At: time.Now(), InstanceConfigETag: instETag} |
| if suffixes, err := relaySuffixesFromRelayVar(m, priv); err != nil { |
| // A CARLOS_RELAY entry that fails to decode is loud on its own — |
| // RenderEnv already refused this render if the failure were a |
diff --git a/internal/appconfig/sync_test.go b/internal/appconfig/sync_test.go| index 2f4c10734..5a0d4263e 100644 |
| --- a/internal/appconfig/sync_test.go |
| +++ b/internal/appconfig/sync_test.go |
| @@ -400,7 +400,7 @@ func TestSyncHostRefusesNonBareHostname(t *testing.T) { |
| } |
| m := &Manifest{Schema: 1, App: "evil", Vars: map[string]Var{"A": {Kind: "env", Value: "1"}}} |
| for _, host := range []string{"../escape", "a/b", "", ".", ".."} { |
| - syncHost(registry.Route{Host: host}, m, false, nil, envDir, func(string) error { return nil }, nil) |
| + syncHost(registry.Route{Host: host}, m, false, nil, envDir, func(string) error { return nil }, "", nil) |
| } |
| if _, err := os.Stat(filepath.Join(envDir, "..", "escape.env")); !os.IsNotExist(err) { |
| t.Fatalf("a path-escaping host wrote outside envDir, stat err = %v", err) |
| @@ -1054,7 +1054,7 @@ func TestSyncHostConvergesAgentOwnedVarWithoutRewrite(t *testing.T) { |
| var restarted int |
| restart := func(string) error { restarted++; return nil } |
| - syncHost(rt, m, false, nil, envDir, restart, nil) |
| + syncHost(rt, m, false, nil, envDir, restart, "", nil) |
| path := filepath.Join(envDir, "steady.example.com.env") |
| got, err := os.ReadFile(path) |
| if err != nil { |
| @@ -1072,7 +1072,7 @@ func TestSyncHostConvergesAgentOwnedVarWithoutRewrite(t *testing.T) { |
| } |
| // Second, identical pass. |
| - syncHost(rt, m, false, nil, envDir, restart, nil) |
| + syncHost(rt, m, false, nil, envDir, restart, "", nil) |
| if restarted != 1 { |
| t.Fatalf("restarted = %d after second identical sync, want still 1 (no re-restart)", restarted) |
| } |
diff --git a/internal/appconfig/syncexec.go b/internal/appconfig/syncexec.go| index 5e520b447..834a10f46 100644 |
| --- a/internal/appconfig/syncexec.go |
| +++ b/internal/appconfig/syncexec.go |
| @@ -37,6 +37,7 @@ import ( |
| "bytes" |
| "context" |
| "crypto/ecdh" |
| + "errors" |
| "fmt" |
| "os" |
| "path/filepath" |
| @@ -362,6 +363,14 @@ func SyncExec(ctx context.Context, reg *registry.Registry, s release.Store, priv |
| // beats an unbounded silent freeze; routes bound to an environment are |
| // protected by the error path regardless, since an unresolved group |
| // never reaches a removal at all. |
| + // |
| + // The instance layer does not enter this count, and does not need to. |
| + // The fault this guard exists for is registry-wide — a mangled account |
| + // column makes EVERY key of every app 404 at once, the overlay listing |
| + // included, since it hangs off the same mangled prefix — so an overlay |
| + // could not survive the fault it is counting. A route that does have an |
| + // overlay never reaches a removal branch at all (see below), so it is |
| + // neither protected nor endangered by this number. |
| eligible, absent := 0, 0 |
| for _, r := range resolved { |
| if r.err != nil { |
| @@ -379,6 +388,21 @@ func SyncExec(ctx context.Context, reg *registry.Registry, s release.Store, priv |
| fmt.Printf(" config sync: all %d resolvable (non-errored) config groups on this box resolved absent — refusing to remove any config files this pass; check the registry's account column and the deployment bucket\n", eligible) |
| } |
| + // The instance layer (senders design §6.2), resolved per route after the |
| + // group base. Same loader, same cost, same failure contract as the unit |
| + // pass's: one listing per app, a Get only for a host it named, and an |
| + // unresolved overlay leaves the file exactly as it is. |
| + overlayFor := instanceOverlayLoader(ctx, s) |
| + // One line per app when the LISTING fails, not one per route: every |
| + // route of that app fails identically and for the same reason, since |
| + // instanceOverlayLoader caches that failure per (account, app) — |
| + // independent of Environment, which is why this is keyed on appKey and |
| + // not groupKey. A per-host Get or parse failure is a DIFFERENT claim — |
| + // two routes can have two separately corrupt overlay objects, each |
| + // wrong for its own reason — so that case is never folded in here; see |
| + // the errors.As branch below. |
| + said := map[appKey]bool{} |
| + |
| for _, rt := range routes { |
| in, addrShape, sidecarShape := inScope(rt) |
| if !in { |
| @@ -405,7 +429,64 @@ func SyncExec(ctx context.Context, reg *registry.Registry, s release.Store, priv |
| fmt.Printf(" %s: config unresolved (environment %q), config file left as-is: %v\n", rt.Host, env, r.err) |
| continue |
| } |
| - if r.m == nil && skipRemovals { |
| + // Fetched here, used at the bottom, because a failure must be able to |
| + // stop a REMOVAL too: deciding a file is orphaned while unable to see |
| + // whether this instance has an overlay would delete a live tenant's |
| + // delivered secrets on a listing blip. |
| + inst, instETag, instErr := overlayFor(k.acct, k.app, rt.Host) |
| + if instErr != nil { |
| + var listErr *instanceOverlayListErr |
| + if errors.As(instErr, &listErr) { |
| + // Said ONCE per (account, app), not once per route: the |
| + // listing this failed on is cached per app inside the |
| + // loader, so every route of the app, across every |
| + // environment it is bound to, fails identically and for |
| + // the same reason; a line each would bury the cause under |
| + // its own repetitions on a box with many routes. |
| + ak := appKey{k.acct, k.app} |
| + if !said[ak] { |
| + said[ak] = true |
| + // rt.Host anchors the line to a real route (the first |
| + // one this pass happened to reach), matching sync.go's |
| + // own wording, so an operator scanning the log has a |
| + // hostname to grep for — every other route of this app |
| + // is silently covered by the same line. |
| + fmt.Printf(" %s/%s: instance config unresolved (seen at %s), config files left as-is: %v\n", k.acct, k.app, rt.Host, instErr) |
| + } |
| + } else { |
| + // A per-host Get or parse failure, unlike the listing |
| + // above, is specific to THIS route: the listing already |
| + // succeeded (it named this host), so another route of |
| + // the same app can have a perfectly good overlay, or a |
| + // failure of its own for a different reason. Folding this |
| + // into the app-level line would leave that route's stale |
| + // config file invisible, so it is reported every time. |
| + fmt.Printf(" %s: instance config unresolved, config file left as-is: %v\n", rt.Host, instErr) |
| + } |
| + continue |
| + } |
| + // An overlay is configuration, so a route that has one is never |
| + // "absent" for the MASS-REMOVAL guard below, however its group base |
| + // resolved: writing a file a bundle asked for is not a removal. |
| + // |
| + // This narrowing (the `inst == nil` clause, added by this branch) |
| + // makes the guard itself strictly narrower than it was before: a |
| + // route with an overlay now sails through where it used to be held |
| + // with everything else. That is backstopped, everywhere this |
| + // narrowing actually matters (r.m == nil, i.e. a mangled account |
| + // column made the group base read absent), by the #344 guard just |
| + // below — EXCEPT for a custom-domain route. release.HostAccountLabel |
| + // only derives a <sqid> to check from a PLATFORM-domain hostname; a |
| + // custom-domain host's label is "", the #344 check's `sq != ""` test |
| + // is false, and the guard simply does not fire. So a custom-domain |
| + // route with an overlay, on a box whose account column is mangled in |
| + // the exact way #344 exists to catch, has no backstop here at all — |
| + // it sails through this guard AND past #344, and the wholesale |
| + // overlay-alone replacement both guards were meant to prevent for it |
| + // can still happen. Known and accepted (final review, plan 4): the |
| + // fix would need a host-independent way to validate the account |
| + // column, which #344 does not have either. |
| + if r.m == nil && inst == nil && skipRemovals { |
| continue |
| } |
| // The #344 guard: an absent manifest removes the config file ONLY |
| @@ -415,6 +496,20 @@ func SyncExec(ctx context.Context, reg *registry.Registry, s release.Store, priv |
| // stamps it from bucket listings, and a mis-stamp deleted a live |
| // tenant's delivered secrets on 2026-08-28). Held, loudly, like an |
| // unresolved group — the fix is the row's account, not this file. |
| + // |
| + // UNLIKE the mass-removal guard above, this hold is NOT narrowed by |
| + // an overlay (review finding, plan 4 task 7 fix round 1: the brief's |
| + // own reasoning — "a route with an overlay never reaches a removal |
| + // branch anyway" — does not transfer here, because this is not a |
| + // removal branch. It is "this account column cannot be trusted, |
| + // touch nothing." A mis-stamp that makes the group's default-bundle |
| + // READ land under the wrong prefix makes the instance layer's own |
| + // WRITE (ApplyInstance) land under that same wrong prefix too, so an |
| + // overlay resolving here under the mis-stamped account is exactly |
| + // what a mis-stamped fleet produces — not evidence the account |
| + // column is safe to act on for this route. Wholesale-replacing the |
| + // file with that overlay's contents would be the 2026-08-28 failure |
| + // again, just reached through the overlay instead of the base. |
| if r.m == nil { |
| if sq := release.HostAccountLabel(rt.Host, appsDomain); sq != "" && sq != k.acct { |
| fmt.Printf(" %s: no config manifest under account %q, but the host names account %q — config file left as-is; fix the route row's account column (#344)\n", |
| @@ -466,7 +561,15 @@ func SyncExec(ctx context.Context, reg *registry.Registry, s release.Store, priv |
| case sidecarShape: |
| hostBounce = nil |
| } |
| - syncExecHost(rt, r.m, Key(k.acct, k.app), priv, dataDir, hostBounce, header, ack) |
| + // The group's merge, overlaid by this instance's own layer. A nil |
| + // base with an overlay is the overlay ALONE, which is a real |
| + // configuration and not the absence that drives a removal; nil and |
| + // nil together stay nil, which still removes the file. |
| + m := r.m |
| + if m != nil || inst != nil { |
| + m = applyInstanceOverlay(k.app, m, inst) |
| + } |
| + syncExecHost(rt, m, Key(k.acct, k.app), priv, dataDir, hostBounce, header, instETag, ack) |
| } |
| // Stale-secrets notice (issue #18 design ruling, ratified 2026-08-03 |
| // review): an addr-shaped route whose app is DEFINITIVELY not opted in |
| @@ -770,6 +873,11 @@ func (r RestartPending) Empty() bool { return len(r.Hosts) == 0 && len(r.Unresol |
| // short list read as a complete one. |
| func AwaitingRestart(ctx context.Context, s release.Store, routes []registry.Route, acct, app, envName string, keys []string) RestartPending { |
| var out RestartPending |
| + // The instance layer, through the same loader the sweeps use, so this |
| + // notice and the pass that writes the files agree about what sits above |
| + // a bundle. One listing for the app, then a Get per route that has an |
| + // overlay; both live for one call, like the caches below. |
| + overlayFor := instanceOverlayLoader(ctx, s) |
| // Both caches exist for the same reason SyncExec's do: a box with many |
| // TCP-addr routes on one bundle must not cost more than one read for |
| // it. Unlike SyncExec's, they live for one call. |
| @@ -841,19 +949,33 @@ func AwaitingRestart(ctx context.Context, s release.Store, routes []registry.Rou |
| if !v.On { |
| continue |
| } |
| - // The shadowing test, and the reason keys is a parameter: a write |
| - // to the DEFAULT bundle changes what a bound route resolves to |
| - // only for keys that route's own bundle does not state. Where every |
| + // The shadowing test, and the reason keys is a parameter: where every |
| // touched key is shadowed, the merged manifest is byte-identical, |
| // syncExecHost's bytes.Equal writes nothing, and naming the route |
| // would send someone to restart a process for a value it was never |
| - // going to see. |
| - // |
| - // Only for a default-bundle write reaching a BOUND route: a write |
| - // to the route's own bundle is never shadowed, and an unbound |
| - // route has nothing above it. |
| - if envName == "" && rt.Environment != "" && len(keys) > 0 && allShadowed(keys, eb.m) { |
| - continue |
| + // going to see. An empty keys disables this whole test (nothing to |
| + // shadow — see keys' own doc comment), so the instance layer is |
| + // consulted only when it could actually change the verdict: a route |
| + // whose overlay is unreadable must not be downgraded to Unresolved |
| + // over a call that was never going to ask it anything. |
| + if len(keys) > 0 { |
| + // Which layers sit ABOVE the bundle that was written, for this |
| + // route. A write cannot change what a route resolves to for a |
| + // key some higher layer states, and the third layer sits above |
| + // both of the other two. |
| + inst, _, instErr := overlayFor(acct, app, rt.Host) |
| + if instErr != nil { |
| + out.Unresolved = append(out.Unresolved, rt.Host) |
| + continue |
| + } |
| + var above []*Manifest |
| + if envName == "" && rt.Environment != "" { |
| + above = append(above, eb.m) |
| + } |
| + above = append(above, inst) |
| + if allShadowed(keys, above...) { |
| + continue |
| + } |
| } |
| out.Hosts = append(out.Hosts, RestartHost{Host: rt.Host, Unit: rt.Unit}) |
| } |
| @@ -865,16 +987,32 @@ func AwaitingRestart(ctx context.Context, s release.Store, routes []registry.Rou |
| return out |
| } |
| -// allShadowed reports whether every one of keys is STATED by m — the test |
| -// that decides whether a default-bundle write can have changed what a route |
| -// bound to m actually resolves to. An unset key counts the same as a set |
| -// one: removing a key the overlay shadows changes the merge by nothing. |
| -func allShadowed(keys []string, m *Manifest) bool { |
| - if m == nil { |
| - return false |
| - } |
| +// allShadowed reports whether every one of keys is STATED by at least one |
| +// of the layers above the bundle a write touched — the test that decides |
| +// whether that write can have changed what a route actually resolves to. An |
| +// unset key counts the same as a set one: removing a key a higher layer |
| +// shadows changes the merge by nothing. |
| +// |
| +// above is variadic because how many layers sit over a write depends on |
| +// which bundle was written: a default-bundle write to a bound route is |
| +// under both the named bundle and the instance overlay, a named-bundle |
| +// write is under the overlay alone, and a write to the overlay itself has |
| +// nothing over it. A nil layer (no bundle of that kind for this route) is |
| +// skipped rather than treated as stating nothing, so an empty list answers |
| +// false and the route is named. |
| +func allShadowed(keys []string, above ...*Manifest) bool { |
| for _, k := range keys { |
| - if _, ok := m.Vars[k]; !ok { |
| + stated := false |
| + for _, m := range above { |
| + if m == nil { |
| + continue |
| + } |
| + if _, ok := m.Vars[k]; ok { |
| + stated = true |
| + break |
| + } |
| + } |
| + if !stated { |
| return false |
| } |
| } |
| @@ -936,6 +1074,11 @@ const execAddrConfigHeader = `# Written by carlos (SyncExec, issue #18). Carlos |
| // unified) — folded into the SAME diff/write/rename path as the vars |
| // themselves, so a header-only change (e.g. its text edited in a future |
| // release) converges exactly like a var change would, once. |
| +// |
| +// instETag is the etag of the instance overlay this route's merge already |
| +// folded in (empty when the route has no overlay), carried through to the |
| +// ack unchanged — see ConfigAck.InstanceConfigETag. |
| +// |
| // ack, if non-nil, is called once this route's file has actually changed |
| // (written, never on removal — see SyncExec's own doc comment) with its |
| // config-adoption acknowledgement (spec §5). Restarted/RestartError come |
| @@ -945,7 +1088,7 @@ const execAddrConfigHeader = `# Written by carlos (SyncExec, issue #18). Carlos |
| // (the tcp-exec-file shape: its process belongs to an external supervisor |
| // this platform never restarts at all, so that shape's cutover is a |
| // member acknowledgement, `carlos relay adopted`, instead). |
| -func syncExecHost(rt registry.Route, m *Manifest, key string, priv *ecdh.PrivateKey, dataDir string, bounce func(host string) error, header string, ack func(ConfigAck)) { |
| +func syncExecHost(rt registry.Route, m *Manifest, key string, priv *ecdh.PrivateKey, dataDir string, bounce func(host string) error, header, instETag string, ack func(ConfigAck)) { |
| if rt.Host == "" || rt.Host == "." || rt.Host == ".." || rt.Host != filepath.Base(rt.Host) { |
| fmt.Printf(" %q: not a bare hostname, refusing to derive a config file path from it\n", rt.Host) |
| return |
| @@ -1043,7 +1186,7 @@ func syncExecHost(rt registry.Route, m *Manifest, key string, priv *ecdh.Private |
| fmt.Printf(" %s: config updated\n", rt.Host) |
| bounceErr := bounceLogged(rt.Host, bounce) |
| if ack != nil { |
| - a := ConfigAck{Host: rt.Host, Shape: shapeOf(rt), At: time.Now()} |
| + a := ConfigAck{Schema: ConfigAckSchema, Host: rt.Host, Shape: shapeOf(rt), At: time.Now(), InstanceConfigETag: instETag} |
| if suffixes, err := relaySuffixesFromRelayVar(m, priv); err != nil { |
| fmt.Printf(" %s: config updated, but CARLOS_RELAY could not be read for the config-ack: %v\n", rt.Host, err) |
| } else { |
diff --git a/internal/appconfig/syncexec_test.go b/internal/appconfig/syncexec_test.go| index 453569c88..6769eceee 100644 |
| --- a/internal/appconfig/syncexec_test.go |
| +++ b/internal/appconfig/syncexec_test.go |
| @@ -773,7 +773,7 @@ func TestSyncExecRefusesNonBareHostname(t *testing.T) { |
| pk, _ := ParsePrivateKey(priv, "test") |
| m := &Manifest{Schema: 1, App: "eleven", Vars: map[string]Var{"A": {Kind: "env", Value: "1"}}} |
| for _, host := range []string{"../escape", "a/b", "", ".", ".."} { |
| - syncExecHost(registry.Route{Host: host}, m, "apps/bab/eleven/config.json", pk, dd, func(string) error { return nil }, "", nil) |
| + syncExecHost(registry.Route{Host: host}, m, "apps/bab/eleven/config.json", pk, dd, func(string) error { return nil }, "", "", nil) |
| } |
| ents, err := os.ReadDir(dd) |
| if err != nil { |
| @@ -2261,6 +2261,75 @@ func TestSyncExecHoldsConfigWhenHostNamesADifferentAccount(t *testing.T) { |
| } |
| } |
| +// The #344 guard is NOT a removal branch, and an overlay does not clear it |
| +// (task 7 fix round 1 — the brief's own reasoning, "a route with an overlay |
| +// never reaches a removal branch anyway," does not transfer to this guard). |
| +// A mis-stamp that makes the group's default-bundle READ land under the |
| +// wrong account prefix makes the instance layer's own WRITE |
| +// (ApplyInstance) land under that same wrong prefix too, so an overlay |
| +// resolving here under the mis-stamped account is exactly what a |
| +// mis-stamped fleet produces — not evidence the account column is safe to |
| +// act on. Sibling of TestSyncExecHoldsConfigWhenHostNamesADifferentAccount, |
| +// with an overlay added to the held route. |
| +func TestSyncExecHoldsConfigWhenHostNamesADifferentAccountEvenWithAnOverlay(t *testing.T) { |
| + ctx, s, reg, dd, priv, _ := execKit(t) |
| + pk, err := ParsePrivateKey(priv, "test") |
| + if err != nil { |
| + t.Fatalf("ParsePrivateKey: %v", err) |
| + } |
| + if err := reg.Upsert(registry.Route{ |
| + Host: "birthday-alarm.bde.oncarlos.test", Kind: "instance", App: "birthday-alarm", |
| + Account: "bab", // the mis-stamp |
| + Status: "ready", Socket: "/run/carlos/birthday.sock", |
| + }); err != nil { |
| + t.Fatalf("Upsert: %v", err) |
| + } |
| + // The overlay lands at apps/bab/birthday-alarm/instances-config/ — the |
| + // SAME mis-stamped prefix the group's absent default bundle was looked |
| + // up under, which is the whole point: a mangled account column mangles |
| + // both. |
| + if _, _, err := ApplyInstance(ctx, s, "bab", "birthday-alarm", "birthday-alarm.bde.oncarlos.test", |
| + map[string]Var{"TENANT": {Kind: "env", Value: "mis-stamped"}}, nil); err != nil { |
| + t.Fatalf("ApplyInstance: %v", err) |
| + } |
| + // A second, healthy group so the mass-removal guard (which needs EVERY |
| + // group absent) stays out of the way — this guard is per-route. |
| + if err := reg.Upsert(registry.Route{ |
| + Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", |
| + Status: "ready", Socket: "/run/carlos/inst.sock", |
| + }); err != nil { |
| + t.Fatalf("Upsert: %v", err) |
| + } |
| + if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"A": "1"}); err != nil { |
| + t.Fatalf("SetEnv: %v", err) |
| + } |
| + path := filepath.Join(dd, "birthday-alarm.bde.oncarlos.test.config.env") |
| + if err := os.WriteFile(path, []byte("BIRTHDAYALARM_INSTANCE_KEY=deadbeef\n"), 0o600); err != nil { |
| + t.Fatalf("stage: %v", err) |
| + } |
| + var bounced []string |
| + out := captureStdout(t, func() { |
| + if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error { |
| + bounced = append(bounced, h) |
| + return nil |
| + }, "oncarlos.test", nil); err != nil { |
| + t.Fatalf("SyncExec: %v", err) |
| + } |
| + }) |
| + cur, err := os.ReadFile(path) |
| + if err != nil || string(cur) != "BIRTHDAYALARM_INSTANCE_KEY=deadbeef\n" { |
| + t.Fatalf("delivered config was replaced by the overlay under a mis-stamped account: %q %v", cur, err) |
| + } |
| + for _, h := range bounced { |
| + if h == "birthday-alarm.bde.oncarlos.test" { |
| + t.Fatalf("held route was bounced: %v", bounced) |
| + } |
| + } |
| + if !strings.Contains(out, "birthday-alarm.bde.oncarlos.test") || !strings.Contains(out, "#344") { |
| + t.Fatalf("the #344 hold did not fire for a route that has an overlay:\n%s", out) |
| + } |
| +} |
| + |
| // TestSyncExecWritesConfigAckForSocketExecRoute is the review's finding 1 |
| // (2026-09-07): syncExecHost DOES bounce a socket-exec route (bounce is |
| // non-nil for it, only a tcp-exec-file/addr route gets nil), so its ack |