| 1 | //go:build linux && rootrunner |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "crypto/ed25519" |
| 8 | "crypto/sha256" |
| 9 | "errors" |
| 10 | "io" |
| 11 | "io/fs" |
| 12 | "net" |
| 13 | "net/http" |
| 14 | "os" |
| 15 | "path/filepath" |
| 16 | "reflect" |
| 17 | "strings" |
| 18 | "testing" |
| 19 | "time" |
| 20 | |
| 21 | "github.com/carlosframework/platform/internal/controlsock" |
| 22 | "github.com/carlosframework/platform/internal/domains" |
| 23 | "github.com/carlosframework/platform/internal/hostbox" |
| 24 | "github.com/carlosframework/platform/internal/hostclaim" |
| 25 | execlease "github.com/carlosframework/platform/internal/hostexec" |
| 26 | "github.com/carlosframework/platform/internal/hostguard" |
| 27 | "github.com/carlosframework/platform/internal/provision" |
| 28 | "github.com/carlosframework/platform/internal/release" |
| 29 | ) |
| 30 | |
| 31 | func lifecycleActiveSibling(t *testing.T) { |
| 32 | t.Helper() |
| 33 | s, err := openClaimRootAuthority() |
| 34 | lifecycleCheck(t, "sibling authority", err) |
| 35 | defer s.Close() |
| 36 | views, err := hostguard.NewRootAuthority(s).GenerationInstallations(t.Context()) |
| 37 | lifecycleCheck(t, "sibling installations", err) |
| 38 | for _, view := range views { |
| 39 | if view.Host != lifecycleSibling { |
| 40 | continue |
| 41 | } |
| 42 | if view.Installation.Stamp == 0 || view.Barrier != nil { |
| 43 | t.Fatal("sibling is not active") |
| 44 | } |
| 45 | for _, old := range view.Retired { |
| 46 | if old.Incarnation == view.Installation.Incarnation && old.Stamp == view.Installation.Stamp { |
| 47 | t.Fatal("sibling retired") |
| 48 | } |
| 49 | } |
| 50 | transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| 51 | return (&net.Dialer{}).DialContext(ctx, "unix", "/run/carlos/instances/"+lifecycleSibling+"/app.sock") |
| 52 | }} |
| 53 | defer transport.CloseIdleConnections() |
| 54 | response, err := (&http.Client{Transport: transport, Timeout: time.Second}).Get("http://sibling/") |
| 55 | lifecycleCheck(t, "active sibling response", err) |
| 56 | defer response.Body.Close() |
| 57 | raw, err := io.ReadAll(response.Body) |
| 58 | lifecycleCheck(t, "active sibling body", err) |
| 59 | if response.StatusCode != http.StatusOK || string(raw) != "lifecycle tenant" { |
| 60 | t.Fatal("sibling stopped serving", response.StatusCode, string(raw)) |
| 61 | } |
| 62 | return |
| 63 | } |
| 64 | t.Fatal("missing production sibling installation") |
| 65 | } |
| 66 | |
| 67 | func lifecycleRetainedBytes(t *testing.T) map[string][32]byte { |
| 68 | t.Helper() |
| 69 | out := map[string][32]byte{} |
| 70 | for _, root := range []string{"/var/lib/carlos-claims", "/data/.parked"} { |
| 71 | lifecycleCheck(t, "retained evidence", filepath.WalkDir(root, func(path string, e fs.DirEntry, err error) error { |
| 72 | if err != nil { |
| 73 | return err |
| 74 | } |
| 75 | if !e.Type().IsRegular() || strings.HasSuffix(path, ".lock") { |
| 76 | return nil |
| 77 | } |
| 78 | raw, err := os.ReadFile(path) |
| 79 | if err != nil { |
| 80 | return err |
| 81 | } |
| 82 | if strings.HasPrefix(path, "/data/.parked/") || strings.Contains(string(raw), lifecycleHost) { |
| 83 | out[path] = sha256.Sum256(raw) |
| 84 | } |
| 85 | return nil |
| 86 | })) |
| 87 | } |
| 88 | return out |
| 89 | } |
| 90 | |
| 91 | func lifecycleReinstall(t *testing.T, store release.Store, key ed25519.PrivateKey, actor hostbox.Actor, pointer release.SystemPointer) { |
| 92 | t.Helper() |
| 93 | lifecycleActiveSibling(t) |
| 94 | removed, err := domains.DetachClaim(t.Context(), store, lifecycleHost, "bab", "lifecycle") |
| 95 | lifecycleCheck(t, "release deleted instance domain", err) |
| 96 | if !removed { |
| 97 | t.Fatal("deleted instance domain was not released") |
| 98 | } |
| 99 | before := lifecycleRetainedBytes(t) |
| 100 | lifecycleCreate(t, store, key, actor, pointer, provision.Record{Schema: 1, Host: lifecycleHost, Account: "bab", App: "reinstall", Channel: "stable", CreatedAt: time.Now().UTC()}) |
| 101 | lifecycleCheck(t, "request reinstall", os.WriteFile("/run/carlos/lifecycle-reinstall", []byte("ready"), 0644)) |
| 102 | deadline := time.Now().Add(time.Minute) |
| 103 | for { |
| 104 | if _, err := os.Stat("/run/carlos/lifecycle-reinstall-refused"); err == nil { |
| 105 | break |
| 106 | } |
| 107 | if time.Now().After(deadline) { |
| 108 | t.Fatal("reinstall refusal timeout") |
| 109 | } |
| 110 | time.Sleep(20 * time.Millisecond) |
| 111 | } |
| 112 | if !reflect.DeepEqual(before, lifecycleRetainedBytes(t)) { |
| 113 | t.Fatal("reinstall changed retained evidence") |
| 114 | } |
| 115 | lifecycleActiveSibling(t) |
| 116 | } |
| 117 | |
| 118 | func lifecycleAgentReinstall(t *testing.T, ctx context.Context, store release.Store) { |
| 119 | t.Helper() |
| 120 | deadline := time.Now().Add(2 * time.Minute) |
| 121 | refused := false |
| 122 | for time.Now().Before(deadline) { |
| 123 | if _, err := os.Stat("/run/carlos/lifecycle-finish"); err == nil { |
| 124 | return |
| 125 | } |
| 126 | if _, err := os.Stat("/run/carlos/lifecycle-reinstall"); err == nil && !refused { |
| 127 | obs, err := hostclaim.Read(ctx, store, lifecycleHost) |
| 128 | lifecycleCheck(t, "reinstall claim", err) |
| 129 | q := executionExpectation(obs.Claim) |
| 130 | q.ID, q.Operation = "reinstall", "executionView" |
| 131 | _, err = rootExecutionClient(controlsock.Path)(ctx, q) |
| 132 | want := lifecycleHost + ": hostname was retired on this box; reinstalling it is not yet supported" |
| 133 | if err == nil || errors.Is(err, execlease.ErrLost) || err.Error() != want { |
| 134 | t.Fatalf("reinstall refusal: %v", err) |
| 135 | } |
| 136 | lifecycleCheck(t, "reinstall refused", os.WriteFile("/run/carlos/lifecycle-reinstall-refused", []byte("ready"), 0644)) |
| 137 | refused = true |
| 138 | } |
| 139 | time.Sleep(20 * time.Millisecond) |
| 140 | } |
| 141 | t.Fatal("lifecycle verification timeout") |
| 142 | } |
| 143 | |
amadan