carlos / platform Public

Download this file

1	package host
2
3	import (
4	"context"
5	"errors"
6	"fmt"
7	"log"
8	"time"
9
10	"github.com/carlosframework/platform/internal/activator"
11	"github.com/carlosframework/platform/internal/registry"
12	)
13
14	// The instance-key backfill (L21, plan 2026-10-10-instance-key-provision.md
15	// ruling 7). Every launch path mints CARLOS_INSTANCE_KEY before it reads the
16	// env, so no process is ever started without one. The backfill gives every
17	// instance already on a box its key now, just after agent start, rather
18	// than at its next start, so the rollout can be checked box by box before
19	// any app adopts the key: a host that the backfill names "present" or
20	// "minted" is ready. It runs in the background once the activator has
21	// started (activatorStart.Run), never on the path the edge waits for.
22	//
23	// It runs only where the legacy activator writes <host>.env itself. With
24	// host claims on, every write to a host's installation runs under the
25	// claim's guard, and the claim's launch source already mints the key under
26	// that guard before every cold admission (claim_snapshot.go
27	// EnsureClaimAdminToken), so a claim box is not backfilled here.
28
29	// instanceKeyBackfiller is the activator surface the backfill needs. The
30	// context bounds the one bucket read a mint makes (ruling 5: nothing is
31	// minted until the row's configuration is delivered).
32	type instanceKeyBackfiller interface {
33	BackfillInstanceKey(context.Context, registry.Route) (activator.InstanceKeyOutcome, error)
34	}
35
36	// instanceKeyBackfillBudget bounds one pass. Each keyless row costs one
37	// read of the bucket's configuration before its key is minted. The pass
38	// runs in the background after the activator starts (activatorStart.Run),
39	// so it never holds the edge up, but with the bucket unreachable the rows
40	// after the budget still fail fast rather than each waiting out its own
41	// read; every one of them mints at its next start.
42	const instanceKeyBackfillBudget = time.Minute
43
44	// hostPreparationSlot runs prepare in host's preparation slot, ordered
45	// with the host's maintenance fence: (*activator.Service).
46	// PrepareScheduledToken. It answers activator.ErrFenced without running
47	// prepare when a fence is held for host, and a fence asked for while
48	// prepare runs is installed only after it returns. Wakes do not wait on
49	// the slot.
50	type hostPreparationSlot func(host string, prepare func() (string, error)) (string, error)
51
52	// errInstanceKeyRowSkipped is a row the backfill listed and, when its turn
53	// came, did not write for: an operation held its host fenced, or its row
54	// had changed (a move, a layout flip, a removal) so it is no longer one
55	// the backfill selects. Counted with the skipped rows; its next start
56	// mints the key if it still needs one.
57	var errInstanceKeyRowSkipped = errors.New("skipped")
58
59	// keyBackfillInSlot is the backfill's step for one row once the activator
60	// is serving (the v0.26.1 hotfix moved the pass off the startup path). The
61	// layout pass and every move fence a host before they move its files, and
62	// install that fence only after any preparation already in the slot has
63	// returned; the slot refuses a host already fenced. So inside the slot no
64	// file of the host moves, and the row is read again there, so the write
65	// goes to the paths the host has now, not those of the list taken before.
66	// A start's own mint is not ordered by the slot: it and the backfill both
67	// decide and append under the activator's adminTokenMu, create-only, so
68	// whichever is second finds the key present.
69	type keyBackfillInSlot struct {
70	keys instanceKeyBackfiller
71	slot hostPreparationSlot
72	route func(string) (*registry.Route, error)
73	}
74
75	func (g keyBackfillInSlot) BackfillInstanceKey(ctx context.Context, rt registry.Route) (activator.InstanceKeyOutcome, error) {
76	got, err := activator.InstanceKeyFailed, error(nil)
77	_, serr := g.slot(rt.Host, func() (string, error) {
78	fresh, rerr := g.route(rt.Host)
79	switch {
80	case rerr != nil:
81	err = fmt.Errorf("read its row: %w", rerr)
82	case fresh == nil || !instanceKeyRoute(*fresh):
83	err = fmt.Errorf("%w: its row changed after the backfill listed it", errInstanceKeyRowSkipped)
84	default:
85	got, err = g.keys.BackfillInstanceKey(ctx, *fresh)
86	}
87	return "", nil
88	})
89	if serr != nil {
90	return activator.InstanceKeyFailed, fmt.Errorf("%w: an operation holds it (%v)", errInstanceKeyRowSkipped, serr)
91	}
92	return got, err
93	}
94
95	// instanceKeyRoute selects the rows the backfill writes for: ready,
96	// activator-spawned instances, the rows execSyncable delivers config to,
97	// without sidecars and without replicas (a replica is never minted a key,
98	// ruling 26). A held row (moving, landing, restoring, releasing) belongs to
99	// its operation and is skipped, as is a provisioning row whose first start
100	// will mint.
101	func instanceKeyRoute(rt registry.Route) bool {
102	return rt.Kind == "instance" && rt.Host != "" && rt.Socket != "" && !rt.Platform && !rt.Replica &&
103	(rt.Status == "" || rt.Status == "ready") &&
104	rt.Backing != registry.BackingUnit && rt.Tunnel == "" && rt.Unit == ""
105	}
106
107	// instanceKeyBackfill counts one pass's outcomes. Deferred is a row whose
108	// configuration sets a key this box has not delivered yet, or whose
109	// configuration could not be read: not minted, by design, and minted or
110	// found configured at its next start.
111	type instanceKeyBackfill struct {
112	Minted, Present, FromConfig, NoEnvFile, Deferred, Failed, Skipped int
113	}
114
115	// backfillInstanceKeys runs the backfill over routes, within
116	// instanceKeyBackfillBudget, and logs one count line when it ends,
117	// "agent: instance keys: ...": on a legacy box that is after the activator
118	// has started, within the budget plus one row's read. It never fails the
119	// agent: a host it cannot write is logged and counted, and that host's
120	// next start mints its key (or refuses, loudly, if the same fault is
121	// still there). Nothing logged names a key.
122	//
123	// The error is read before the outcome: a refused or failed row answers
124	// InstanceKeyFailed (the zero outcome) beside its error, and counting an
125	// outcome first would call a host ready whose key was never written.
126	func backfillInstanceKeys(ctx context.Context, routes []registry.Route, b instanceKeyBackfiller) instanceKeyBackfill {
127	ctx, cancel := context.WithTimeout(ctx, instanceKeyBackfillBudget)
128	defer cancel()
129	var st instanceKeyBackfill
130	for _, rt := range routes {
131	if ctx.Err() != nil {
132	break
133	}
134	if !instanceKeyRoute(rt) {
135	st.Skipped++
136	continue
137	}
138	got, err := b.BackfillInstanceKey(ctx, rt)
139	switch {
140	case errors.Is(err, errInstanceKeyRowSkipped):
141	st.Skipped++
142	log.Printf("agent: instance key for %s: %v (its next start mints it)", rt.Host, err)
143	continue
144	case errors.Is(err, activator.ErrInstanceKeyDeferred):
145	st.Deferred++
146	log.Printf("agent: instance key for %s: %v", rt.Host, err)
147	continue
148	case err != nil:
149	st.Failed++
150	log.Printf("agent: instance key for %s: %v (its next start mints it, or refuses for the same reason)", rt.Host, err)
151	continue
152	}
153	switch got {
154	case activator.InstanceKeyMinted:
155	st.Minted++
156	case activator.InstanceKeyPresent:
157	st.Present++
158	case activator.InstanceKeyFromConfig:
159	st.FromConfig++
160	case activator.InstanceKeyNoEnvFile:
161	st.NoEnvFile++
162	case activator.InstanceKeyReplica:
163	// Unreachable: instanceKeyRoute drops every replica row. Kept
164	// so the counts always sum to the rows listed.
165	st.Skipped++
166	}
167	}
168	log.Printf("agent: instance keys: %d minted, %d already present, %d set by app configuration, %d never started here, %d waiting for their configuration, %d failed, %d rows skipped",
169	st.Minted, st.Present, st.FromConfig, st.NoEnvFile, st.Deferred, st.Failed, st.Skipped)
170	return st
171	}
172