carlos / platform Public

Download this file

1	package host
2
3	import (
4	"encoding/json"
5	"errors"
6	"os"
7	"os/exec"
8	"path/filepath"
9	"testing"
10	"time"
11
12	"github.com/carlosframework/platform/internal/hostwrite"
13	"github.com/carlosframework/platform/internal/registry"
14	"github.com/carlosframework/platform/internal/release"
15	"github.com/carlosframework/platform/internal/rootstate"
16	)
17
18	func TestPlan14ActiveRecoveryProcessCrash(t *testing.T) {
19	if boundary := os.Getenv("PLAN14_ACTIVE_CRASH"); boundary != "" {
20	dir := os.Getenv("PLAN14_ACTIVE_DIR")
21	e, u, c, source, pub := signedRecoveryFixture(t)
22	storeDir := filepath.Join(dir, "deployment")
23	store := release.NewDirStore(storeDir)
24	keys, err := e.store.List(t.Context(), "")
25	if err != nil {
26	t.Fatal(err)
27	}
28	for _, key := range keys {
29	raw, _, err := e.store.Get(t.Context(), key)
30	if err != nil {
31	t.Fatal(err)
32	}
33	if _, err := store.Put(t.Context(), key, raw, ""); err != nil {
34	t.Fatal(err)
35	}
36	}
37	source.store = store
38	disk := &rollbackDisk{dir: filepath.Join(dir, "attempts")}
39	if err := os.Mkdir(disk.dir, 0700); err != nil {
40	t.Fatal(err)
41	}
42	c.attempts = func() (rootstate.Store, error) { return disk, nil }
43	crash := func() {
44	raw, err := json.Marshal(rollbackProcessFixture{Props: u.props, Exe: u.exe, Opt: u.opt, Store: storeDir, Effects: u.effects})
45	if err != nil {
46	t.Fatal(err)
47	}
48	if err := os.WriteFile(filepath.Join(dir, "fixture.json"), raw, 0600); err != nil {
49	t.Fatal(err)
50	}
51	if err := os.WriteFile(filepath.Join(dir, "public-key"), []byte(os.Getenv("CARLOS_RELEASE_PUBKEY")), 0600); err != nil {
52	t.Fatal(err)
53	}
54	os.Exit(75)
55	}
56	disk.afterWrite = func(k string, raw []byte) {
57	if k != rollbackJournalKey(observedHost) {
58	return
59	}
60	var a rollbackAttempt
61	if json.Unmarshal(raw, &a) != nil {
62	t.Fatal("bad attempt")
63	}
64	if boundary == "intent" && !a.Failed && !a.Recovering && !a.Settled {
65	crash()
66	}
67	if boundary == "rollback pending" && a.Recovering && !a.Settled {
68	crash()
69	}
70	if boundary == "verified" && a.Verified && a.Settled {
71	crash()
72	}
73	}
74	u.crashNew = boundary == "forward failed" || boundary == "rollback pending" || boundary == "rollback running"
75	u.afterEffect = func() {
76	if len(u.effects) == 1 && (boundary == "forward running" || boundary == "forward failed") {
77	crash()
78	}
79	if len(u.effects) == 3 && boundary == "rollback running" {
80	crash()
81	}
82	}
83	n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink}
84	_, err = n.recover(t.Context(), observedHost, c, func(string) string { return "5s" }, e.root.reg)
85	t.Fatalf("crash boundary not reached: %v", err)
86	}
87	for _, boundary := range []string{"intent", "forward running", "forward failed", "rollback pending", "rollback running", "verified"} {
88	t.Run(boundary, func(t *testing.T) {
89	dir := t.TempDir()
90	cmd := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestPlan14ActiveRecoveryProcessCrash$", "-test.count=1")
91	cmd.Dir = dir
92	cmd.Env = append(os.Environ(), "TMPDIR="+dir, "PLAN14_ACTIVE_CRASH="+boundary, "PLAN14_ACTIVE_DIR="+dir, "CARLOS_HOST_CLAIM=")
93	out, err := cmd.CombinedOutput()
94	var exit *exec.ExitError
95	if !errors.As(err, &exit) || exit.ExitCode() != 75 {
96	t.Fatalf("child: %v\n%s", err, out)
97	}
98	raw, err := os.ReadFile(filepath.Join(dir, "fixture.json"))
99	if err != nil {
100	t.Fatal(err)
101	}
102	var f rollbackProcessFixture
103	if err := json.Unmarshal(raw, &f); err != nil {
104	t.Fatal(err)
105	}
106	key, err := os.ReadFile(filepath.Join(dir, "public-key"))
107	if err != nil {
108	t.Fatal(err)
109	}
110	t.Setenv("CARLOS_RELEASE_PUBKEY", string(key))
111	u := &plan14RestartUnits{props: f.Props, exe: f.Exe, opt: f.Opt, effects: f.Effects}
112	source := &proofBindingSource{store: release.NewDirStore(f.Store), sys: u, optDir: u.opt, openFile: os.Open, unitDirs: []string{filepath.Dir(u.props.FragmentPath)}}
113	disk := &rollbackDisk{dir: filepath.Join(dir, "attempts")}
114	a, err := readRollbackAttempt(disk, observedHost)
115	if err != nil {
116	t.Fatal(err)
117	}
118	now := a.Created.Add(time.Second)
119	c := rootRestartChecks{attempts: func() (rootstate.Store, error) { return disk, nil }, host: source.forHost, rollback: source.rollback, prepareRollback: source.prepareRollback, recovered: source.recovered, ready: func(registry.Route) func() error { return func() error { return nil } }, now: func() time.Time { return now }, sleep: func(d time.Duration) { now = now.Add(d) }, history: func() (*release.RootHistoryPass, error) {
120	return nil, os.ErrPermission
121	}}
122	before := len(u.effects)
123	if err := recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}); err != nil {
124	t.Fatal(err)
125	}
126	want := "v1"
127	if boundary == "forward running" || boundary == "verified" {
128	want = "v2"
129	}
130	if u.exe != release.VersionPath(u.opt, "bab", want, "console") || u.props.MainPID <= 0 {
131	t.Fatalf("wrong recovered process: %+v", u)
132	}
133	if boundary != "forward failed" && boundary != "rollback pending" && len(u.effects) != before {
134	t.Fatalf("effect repeated: %v", u.effects)
135	}
136	before = len(u.effects)
137	if err := recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}); err != nil || len(u.effects) != before {
138	t.Fatalf("replay: %v %v", err, u.effects)
139	}
140	})
141	}
142	}
143