carlos / platform Public

Download this file

1	package hostops
2
3	import (
4	"context"
5	"crypto/ed25519"
6	"crypto/rand"
7	"encoding/json"
8	"errors"
9	"fmt"
10	"strings"
11	"testing"
12	"time"
13
14	"github.com/carlosframework/platform/internal/hostactors"
15	"github.com/carlosframework/platform/internal/hostbox"
16	"github.com/carlosframework/platform/internal/hostclaim"
17	"github.com/carlosframework/platform/internal/hostevidence"
18	"github.com/carlosframework/platform/internal/lease"
19	"github.com/carlosframework/platform/internal/lease/leasetest"
20	"github.com/carlosframework/platform/internal/park"
21	"github.com/carlosframework/platform/internal/release"
22	)
23
24	func TestAdoptedHostDiscoveryThroughProductionRegionalWorker(t *testing.T) {
25	ctx := context.Background()
26	pub, key, err := ed25519.GenerateKey(rand.Reader)
27	if err != nil {
28	t.Fatal(err)
29	}
30	boxPub, boxKey, err := ed25519.GenerateKey(rand.Reader)
31	if err != nil {
32	t.Fatal(err)
33	}
34	store := release.NewDirStore(t.TempDir())
35	at := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC)
36	enrollment := strings.Repeat("a", 32)
37	roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 3, RequestLimitNS: int64(hostbox.RequestLimit), Regions: map[string]hostactors.Region{"ie": {}, "uk": {}}, Boxes: map[string]hostactors.Box{
38	"box": {CensusPublicKey: boxPub, Enrollment: enrollment, Namespace: "platform", BoxID: "box", Label: "box", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "ie", InstanceID: "i-box"}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"},
39	}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}}
40	live := roster.Boxes["box"]
41	live.BoxID = "live"
42	live.Label = "live"
43	live.Region = "uk"
44	live.Enrollment = strings.Repeat("b", 32)
45	live.Cloud.Region = "uk"
46	live.Cloud.InstanceID = "i-live"
47	roster.Boxes["live"] = live
48	signed, err := release.SignRootEnvelope(release.RootSigner(key), roster)
49	if err != nil {
50	t.Fatal(err)
51	}
52	if _, err = store.Put(ctx, hostactors.Key, signed, ""); err != nil {
53	t.Fatal(err)
54	}
55	host := "adopted.example"
56	nh := hostclaim.NewHolder{Acquisition: "acquisition", Family: "instance", Key: "apps/acct/app/instances/" + host + ".json", Account: "acct", App: "app", Host: host, CreatedAt: at, Region: "uk"}
57	seq := 0
58	next := func() string { seq++; return fmt.Sprintf("id%d", seq) }
59	obs, err := hostclaim.Update(ctx, store, hostclaim.Observed{Host: host, Absent: true}, hostclaim.Adopt(nh, "live", []string{"ie"}, []string{"box"}, "discovery", next))
60	if err != nil {
61	t.Fatal(err)
62	}
63	// Completing the local chore must not erase the non-serving writer census.
64	for _, ob := range obs.Claim.Obligations {
65	if ob.Kind == "park-local" {
66	obs, err = hostclaim.Update(ctx, store, obs, hostclaim.CompleteObligation(ob.Op, ob.ID))
67	if err != nil {
68	t.Fatal(err)
69	}
70	}
71	}
72	inc := obs.Claim.Holder.Incarnation
73	receipt := StopReceipt{V: 1, ID: "empty", Host: host, Incarnation: inc, OperationID: "discovery", Phase: "discovery", ObligationID: "box-census", Box: "box", Enrollment: enrollment, BootID: "boot", ControllerID: "controller", ClaimGen: obs.Claim.Gen, DirectoryRevision: "3", Kind: "empty", EvidenceAt: at, FenceID: "census-fence", InventorySHA256: strings.Repeat("d", 64), LocalJournalSHA256: strings.Repeat("e", 64), ProducerDrainDigest: strings.Repeat("f", 64), RequestLimitNS: int64(hostbox.RequestLimit)}
74	raw, err := json.Marshal(receipt)
75	if err != nil {
76	t.Fatal(err)
77	}
78	if err = hostevidence.Publish(ctx, store, boxKey, "control/hosts/"+host+"/evidence/discovery/box/empty.json", raw); err != nil {
79	t.Fatal(err)
80	}
81	authority := ProductionAuthority{Store: store, Directory: hostactors.Directory{Store: store, PublicKey: pub, Bucket: "bucket"}, Now: func() time.Time { return at.Add(time.Hour) }}
82	bucket := leasetest.NewVersionedBucket()
83	bucket.AddBucket("data")
84	checks := 0
85	regional := hostbox.Regional{Owner: func(context.Context, hostclaim.Observed) error { return nil }, Engine: park.Engine{
86	Store: hostbox.RequestStore{Store: bucket, Check: func(context.Context) error { return nil }}, Claims: store, Bucket: "data", Region: "ie", Leases: discoveryLeases{}, Now: authority.Now, NewID: next,
87	View: func(_ context.Context, o hostclaim.Observed) (park.View, error) {
88	return park.View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "uk"}, nil
89	},
90	Authorize: func(_ context.Context, _ hostclaim.Observed, ob hostclaim.Obligation) error {
91	checks++
92	if ob.Park == nil {
93	return ErrIdentity
94	}
95	return nil
96	},
97	}}
98	worker, err := NewClaimRegionalWorker(regional, discoveryEnrollment{}, authority)
99	if err != nil {
100	t.Fatal(err)
101	}
102	hosts, err := worker.Census(ctx)
103	if err != nil || len(hosts) != 1 || hosts[0] != host {
104	t.Fatalf("census=%v %v", hosts, err)
105	}
106	// At-rest sweeping may remain pending; discovery itself must complete.
107	if err = worker.Run(ctx); err != nil && !errors.Is(err, ErrPending) {
108	t.Fatal(err)
109	}
110	after, err := hostclaim.Read(ctx, store, host)
111	if err != nil {
112	t.Fatal(err)
113	}
114	if len(after.Claim.Obligations) != 0 || !after.Claim.Touched.Complete || checks == 0 {
115	t.Fatalf("discovery stuck: %+v checks=%d", after.Claim, checks)
116	}
117	if after.Claim.Op.Exec != "live" || after.Claim.Op.Kind != hostclaim.KindServing {
118	t.Fatal("discovery stopped live serving holder")
119	}
120	// A pre-bound Moves park keeps its own prior verifier. It must not ask
121	// the live destination for a delete-style stop receipt.
122	move := hostclaim.Obligation{ID: "move-park", Op: "move", Kind: "park-replica", Region: "ie", Owner: "region:ie", Retirement: "move-retire", Park: &hostclaim.ParkScope{Location: "moves/" + host + "/move", Incarnation: inc, EvidenceAt: at, RequestLimit: hostbox.RequestLimit}}
123	after, err = hostclaim.Update(ctx, store, after, func(c *hostclaim.Claim) error { c.Obligations = append(c.Obligations, move); return nil })
124	if err != nil {
125	t.Fatal(err)
126	}
127	deny := true
128	moveChecks := 0
129	regional.Engine.Authorize = func(_ context.Context, _ hostclaim.Observed, ob hostclaim.Obligation) error {
130	moveChecks++
131	if deny || ob.Op != "move" || ob.Park == nil || *ob.Park != *move.Park {
132	return ErrIdentity
133	}
134	return nil
135	}
136	worker, err = NewClaimRegionalWorker(regional, discoveryEnrollment{}, authority)
137	if err != nil {
138	t.Fatal(err)
139	}
140	if err = worker.Run(ctx); !errors.Is(err, ErrIdentity) {
141	t.Fatalf("move verifier bypassed: %v", err)
142	}
143	after, err = hostclaim.Read(ctx, store, host)
144	if err != nil || len(after.Claim.Obligations) != 1 {
145	t.Fatalf("denied move changed chore: %v", err)
146	}
147	deny = false
148	if err = worker.Run(ctx); err != nil && !errors.Is(err, ErrPending) {
149	t.Fatal(err)
150	}
151	after, err = hostclaim.Read(ctx, store, host)
152	if err != nil || len(after.Claim.Obligations) != 0 || moveChecks < 2 {
153	t.Fatalf("bound move stuck: %v checks=%d", err, moveChecks)
154	}
155	generation := hostclaim.Obligation{ID: "generation-park", Op: "session:1", Kind: "park-generation", Region: "ie", Generation: "G1", Owner: "region:ie", Retirement: "generation-retire"}
156	scope := hostclaim.ParkScope{Location: after.Claim.Holder.Location, Incarnation: inc, EvidenceAt: at, RequestLimit: hostbox.RequestLimit}
157	after, err = hostclaim.Update(ctx, store, after, func(c *hostclaim.Claim) error { c.Obligations = append(c.Obligations, generation); return nil })
158	if err != nil {
159	t.Fatal(err)
160	}
161	regional.Engine.Authorize = func(_ context.Context, _ hostclaim.Observed, ob hostclaim.Obligation) error {
162	if ob.ID != generation.ID || ob.Op != generation.Op || ob.Generation != generation.Generation || ob.Park == nil || *ob.Park != scope {
163	return ErrIdentity
164	}
165	return nil
166	}
167	worker, err = NewClaimRegionalWorker(regional, discoveryEnrollment{}, authority)
168	if err != nil {
169	t.Fatal(err)
170	}
171	if err = worker.Run(ctx); !errors.Is(err, ErrContract) {
172	t.Fatalf("unbound old chore accepted without its verifier: %v", err)
173	}
174	after, err = hostclaim.Read(ctx, store, host)
175	if err != nil || len(after.Claim.Obligations) != 1 || after.Claim.Obligations[0].Park != nil {
176	t.Fatalf("missing verifier changed chore: %v", err)
177	}
178	fallback := &discoveryPreviousEvidence{Host: host, Obligation: generation, Scope: scope}
179	worker, err = NewClaimRegionalWorker(regional, discoveryEnrollment{}, authority, fallback)
180	if err != nil {
181	t.Fatal(err)
182	}
183	if err = worker.Run(ctx); err != nil && !errors.Is(err, ErrPending) {
184	t.Fatal(err)
185	}
186	after, err = hostclaim.Read(ctx, store, host)
187	if err != nil || len(after.Claim.Obligations) != 0 || fallback.ObligationCalls != 1 || fallback.RetirementCalls != 1 {
188	t.Fatalf("lost previous verifier: %v obligation=%d retirement=%d", err, fallback.ObligationCalls, fallback.RetirementCalls)
189	}
190	// Either marker selects the new evidence contract; neither can silently
191	// become an old-generation request, even with a previous verifier present.
192	provider := worker.Evidence.(hostbox.RegionalObligationEvidence)
193	for _, marked := range []hostclaim.Obligation{
194	{ID: "marked", Op: "delete", Kind: "park-replica", Region: "ie", Owner: "region:ie", ParkTarget: scope.Location},
195	{ID: "marked", Op: "delete", Kind: "park-replica", Region: "ie", Owner: "region:ie", EvidenceRefs: map[string]hostclaim.RetirementEvidenceRef{"box": {Box: "box"}}},
196	} {
197	if _, err = provider.VerifyObligation(ctx, after, marked); err == nil {
198	t.Fatal("marked retirement borrowed old evidence")
199	}
200	}
201	if fallback.ObligationCalls != 1 {
202	t.Fatal("marked work reached old verifier")
203	}
204	}
205
206	type discoveryPreviousEvidence struct {
207	Host string
208	Obligation hostclaim.Obligation
209	Scope hostclaim.ParkScope
210	ObligationCalls, RetirementCalls int
211	}
212
213	func (e *discoveryPreviousEvidence) VerifyObligation(_ context.Context, obs hostclaim.Observed, ob hostclaim.Obligation) (hostclaim.ParkScope, error) {
214	e.ObligationCalls++
215	want := e.Obligation
216	if obs.Host != e.Host || obs.Claim == nil || obs.Claim.Holder == nil || obs.Claim.Holder.Incarnation != e.Scope.Incarnation || ob.ID != want.ID || ob.Op != want.Op || ob.Kind != want.Kind || ob.Generation != want.Generation || ob.Region != want.Region || ob.Owner != want.Owner || ob.Retirement != want.Retirement || ob.Park != nil || ob.ParkTarget != "" || len(ob.EvidenceRefs) != 0 {
217	return hostclaim.ParkScope{}, ErrIdentity
218	}
219	return e.Scope, nil
220	}
221	func (e *discoveryPreviousEvidence) VerifyRetirement(_ context.Context, obs hostclaim.Observed, _ []lease.ObjectVersion) (string, time.Time, error) {
222	e.RetirementCalls++
223	if obs.Host != e.Host {
224	return "", time.Time{}, ErrIdentity
225	}
226	return "", time.Time{}, ErrPending
227	}
228
229	type discoveryEnrollment struct{}
230
231	func (discoveryEnrollment) VerifyRegional(context.Context, string, string, time.Duration) error {
232	return nil
233	}
234
235	type discoveryLease struct{}
236
237	func (discoveryLease) Check(context.Context) error { return nil }
238	func (discoveryLease) Release(context.Context) error { return nil }
239	func (discoveryLease) Close() {}
240
241	type discoveryLeases struct{}
242
243	func (discoveryLeases) Acquire(context.Context, string, string) (park.HeldLease, error) {
244	return discoveryLease{}, nil
245	}
246