carlos / platform Public

Download this file

1	package host
2
3	import (
4	"context"
5	"errors"
6	"os"
7	"path/filepath"
8	"testing"
9
10	"github.com/carlosframework/platform/internal/appconfig"
11	"github.com/carlosframework/platform/internal/hostwrite"
12	"github.com/carlosframework/platform/internal/provision"
13	"github.com/carlosframework/platform/internal/registry"
14	"github.com/carlosframework/platform/internal/release"
15	)
16
17	// Harness Run 9, finding 6: a nested instance created with settings has no
18	// instance directory until its first wake, so config sync's write of its
19	// <host>.config.env failed, and only the hourly pass came back for it. The
20	// instance woke and ran without its settings (its store binding included).
21	//
22	// Through the agent's own wiring: the boot or hourly pass (syncExecSerialized)
23	// skips the row and names it, while a sibling whose directory exists gets its
24	// file in the same pass; the generation loop's tick (configGenTick) does
25	// nothing while the directory is absent, and on the first tick after the
26	// first wake has made it, writes the file.
27	func TestConfigGenTickWritesANestedRowsConfigOnceItsDirAppears(t *testing.T) {
28	configDirsPending = pendingDirs{}
29	t.Cleanup(func() { configDirsPending = pendingDirs{} })
30	ctx := context.Background()
31	s := release.NewDirStore(t.TempDir())
32	reg := restartTestRegistry(t)
33	dd := t.TempDir()
34	privText, _, err := appconfig.GenerateKey()
35	if err != nil {
36	t.Fatal(err)
37	}
38	priv, err := appconfig.ParsePrivateKey(privText, "test")
39	if err != nil {
40	t.Fatal(err)
41	}
42	if err := appconfig.SetEnv(ctx, s, "bab", "shop", map[string]string{"STORE_BUCKET": "store-shop"}); err != nil {
43	t.Fatal(err)
44	}
45	fresh, woken := "shop.example.com", "older.example.com"
46	for _, h := range []string{fresh, woken} {
47	if err := reg.Upsert(registry.Route{Host: h, Kind: "instance", App: "shop", Account: "bab", Hibernate: true,
48	Status: "ready", DBPath: provision.NestedDBPath(dd, h), Socket: provision.NestedSocket(h)}); err != nil {
49	t.Fatal(err)
50	}
51	}
52	// The sibling has woken before: the activator's prepare made its
53	// instance directory (os.MkdirAll(dataDir, 0o700), activator/prepare.go).
54	firstWake := func(host string) {
55	t.Helper()
56	if err := os.MkdirAll(provision.InstanceDir(dd, host), 0o700); err != nil {
57	t.Fatal(err)
58	}
59	}
60	firstWake(woken)
61	cfgPath := func(h string) string { return filepath.Join(provision.InstanceDir(dd, h), h+".config.env") }
62
63	// The boot pass (configSyncLoop's first, a minute after start).
64	if err := syncExecSerialized(ctx, reg, s, priv, dd, nil, hostwrite.Config{}); err != nil {
65	t.Fatalf("a nested row with no directory failed the pass: %v", err)
66	}
67	if b, err := os.ReadFile(cfgPath(woken)); err != nil || string(b) != "STORE_BUCKET=store-shop\n" {
68	t.Fatalf("the sibling's bundle = %q, %v", b, err)
69	}
70	if _, err := os.Lstat(provision.InstanceDir(dd, fresh)); !os.IsNotExist(err) {
71	t.Fatalf("config sync made the instance directory itself (err %v): that is the activator's", err)
72	}
73
74	// The generation loop: the first tick seeds its cache, and no tick
75	// writes while the directory is absent.
76	cache := newGenCache()
77	tick := func() {
78	t.Helper()
79	routes, err := reg.List()
80	if err != nil {
81	t.Fatal(err)
82	}
83	configGenTick(ctx, routes, reg, s, priv, dd, nil, hostwrite.Config{}, cache)
84	}
85	for range 2 {
86	tick()
87	}
88	if _, err := os.Lstat(cfgPath(fresh)); !os.IsNotExist(err) {
89	t.Fatalf("a bundle with nowhere to go: %v", err)
90	}
91
92	// The first wake makes the directory; the next tick writes the bundle,
93	// with no config change and no hourly pass.
94	firstWake(fresh)
95	tick()
96	if b, err := os.ReadFile(cfgPath(fresh)); err != nil || string(b) != "STORE_BUCKET=store-shop\n" {
97	t.Fatalf("after the first wake, the bundle = %q, %v", b, err)
98	}
99	if _, ok := configDirsPending.appeared(); ok {
100	t.Fatal("the written host is still waited for")
101	}
102	// Nothing more to wait for: a later tick does not run the pass again.
103	if err := os.Remove(cfgPath(fresh)); err != nil {
104	t.Fatal(err)
105	}
106	tick()
107	if _, err := os.Lstat(cfgPath(fresh)); !os.IsNotExist(err) {
108	t.Fatalf("a tick with nothing pending ran the pass (err %v)", err)
109	}
110	}
111
112	// A pass that fails part way does not forget a host an earlier pass was
113	// waiting for; a complete pass that no longer has the host in scope drops it.
114	func TestConfigDirsPendingKeepsWhatAFailedPassDidNotReach(t *testing.T) {
115	var p pendingDirs
116	dir := filepath.Join(t.TempDir(), "instances", "shop.example.com")
117	p.record(map[string]string{"shop.example.com": dir}, nil, true)
118	p.record(map[string]string{}, nil, false)
119	if err := os.MkdirAll(dir, 0o700); err != nil {
120	t.Fatal(err)
121	}
122	if h, ok := p.appeared(); !ok || h != "shop.example.com" {
123	t.Fatalf("appeared = %q %v after a failed pass", h, ok)
124	}
125	p.record(map[string]string{}, map[string]bool{}, true)
126	if h, ok := p.appeared(); ok {
127	t.Fatalf("appeared = %q after a complete pass with the host out of scope", h)
128	}
129	}
130
131	// blipStore fails reads of one key while fail is set: a bucket blip on
132	// one app's bundle.
133	type blipStore struct {
134	release.Store
135	key string
136	fail bool
137	}
138
139	func (f *blipStore) Get(ctx context.Context, key string) ([]byte, string, error) {
140	if f.fail && key == f.key {
141	return nil, "", errors.New("transient: connection reset")
142	}
143	return f.Store.Get(ctx, key)
144	}
145
146	// Fix round 1, Astra's finding 3: the first tick after the first wake makes
147	// the directory meets a transient failure reading the app's bundle. The pass
148	// returns nil, leaving the file unwritten; the host stays waited for, and the
149	// next tick writes its config, with no config change and no hourly pass. A
150	// bundle that keeps failing is retried on a backoff, not every tick.
151	func TestConfigGenTickRetriesAHostAPassLeftUndone(t *testing.T) {
152	configDirsPending = pendingDirs{}
153	t.Cleanup(func() { configDirsPending = pendingDirs{} })
154	ctx := context.Background()
155	base := release.NewDirStore(t.TempDir())
156	s := &blipStore{Store: base, key: appconfig.Key("bab", "shop")}
157	reg := restartTestRegistry(t)
158	dd := t.TempDir()
159	privText, _, err := appconfig.GenerateKey()
160	if err != nil {
161	t.Fatal(err)
162	}
163	priv, err := appconfig.ParsePrivateKey(privText, "test")
164	if err != nil {
165	t.Fatal(err)
166	}
167	if err := appconfig.SetEnv(ctx, base, "bab", "shop", map[string]string{"STORE_BUCKET": "store-shop"}); err != nil {
168	t.Fatal(err)
169	}
170	host := "shop.example.com"
171	if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", App: "shop", Account: "bab", Hibernate: true,
172	Status: "ready", DBPath: provision.NestedDBPath(dd, host), Socket: provision.NestedSocket(host)}); err != nil {
173	t.Fatal(err)
174	}
175	cfgPath := filepath.Join(provision.InstanceDir(dd, host), host+".config.env")
176	cache := newGenCache()
177	passes := 0
178	tick := func() {
179	t.Helper()
180	routes, err := reg.List()
181	if err != nil {
182	t.Fatal(err)
183	}
184	if _, due := configDirsPending.peek(); due {
185	passes++
186	}
187	configGenTick(ctx, routes, reg, s, priv, dd, nil, hostwrite.Config{}, cache)
188	}
189	if err := syncExecSerialized(ctx, reg, s, priv, dd, nil, hostwrite.Config{}); err != nil {
190	t.Fatal(err)
191	}
192	tick() // seeds the cache
193	if err := os.MkdirAll(provision.InstanceDir(dd, host), 0o700); err != nil {
194	t.Fatal(err)
195	}
196	s.fail = true
197	tick()
198	if _, err := os.Lstat(cfgPath); !os.IsNotExist(err) {
199	t.Fatalf("a bundle that did not read was written (err %v)", err)
200	}
201	if _, ok := configDirsPending.peek(); !ok {
202	t.Fatal("the pass that left the host undone dropped it")
203	}
204	s.fail = false
205	tick()
206	if b, err := os.ReadFile(cfgPath); err != nil || string(b) != "STORE_BUCKET=store-shop\n" {
207	t.Fatalf("the next tick's bundle = %q, %v", b, err)
208	}
209	if _, ok := configDirsPending.peek(); ok {
210	t.Fatal("the written host is still waited for")
211	}
212
213	// A bundle that keeps failing: retries back off (1, 2, 4 ... ticks apart).
214	if err := os.Remove(cfgPath); err != nil {
215	t.Fatal(err)
216	}
217	configDirsPending.record(map[string]string{host: provision.InstanceDir(dd, host)}, nil, false)
218	s.fail = true
219	passes = 0
220	for range 20 {
221	tick()
222	}
223	if passes == 0 || passes > 6 {
224	t.Fatalf("%d retries in 20 ticks of a failing bundle, want a backoff (1 to 6)", passes)
225	}
226	}
227
228	// peek is appeared without the tick: whether a retry is due now.
229	func (p *pendingDirs) peek() (string, bool) {
230	p.mu.Lock()
231	defer p.mu.Unlock()
232	for h, e := range p.dirs {
233	if _, err := os.Lstat(e.dir); err == nil && e.waitTicks == 0 {
234	return h, true
235	}
236	}
237	return "", false
238	}
239
240	// TestPendingDirsBackoffSaturates: a host that keeps failing backs off to
241	// the cap and stays there; the failure count never grows far enough for the
242	// shift to overflow into a negative or zero delay (Astra fix round 1).
243	func TestPendingDirsBackoffSaturates(t *testing.T) {
244	var p pendingDirs
245	dir := t.TempDir()
246	p.record(map[string]string{"shop.example.com": dir}, nil, false)
247	for i := 0; i < 200; i++ {
248	p.record(nil, map[string]bool{"shop.example.com": false}, false)
249	e := p.dirs["shop.example.com"]
250	if e == nil {
251	t.Fatalf("pass %d: the waiting host was dropped", i)
252	}
253	if e.waitTicks < 0 || e.waitTicks > pendingRetryCap-1 {
254	t.Fatalf("pass %d: waitTicks %d outside [0,%d]", i, e.waitTicks, pendingRetryCap-1)
255	}
256	}
257	if got := p.dirs["shop.example.com"].waitTicks; got != pendingRetryCap-1 {
258	t.Fatalf("after 200 failures waitTicks = %d, want the cap %d", got, pendingRetryCap-1)
259	}
260	p.record(nil, map[string]bool{"shop.example.com": true}, false)
261	if _, ok := p.dirs["shop.example.com"]; ok {
262	t.Fatal("a delivered host stays pending")
263	}
264	}
265