amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux && rootrunner
2
3package host
4
5import (
6 "context"
7 "crypto/ed25519"
8 "crypto/sha256"
9 "encoding/hex"
10 "encoding/json"
11 "net"
12 "os"
13 "os/exec"
14 "os/user"
15 "path/filepath"
16 "strconv"
17 "strings"
18 "testing"
19 "time"
20
21 "github.com/carlosframework/platform/internal/cgroup"
22 "github.com/carlosframework/platform/internal/controlsock"
23 "github.com/carlosframework/platform/internal/hostactors"
24 "github.com/carlosframework/platform/internal/hostbox"
25 "github.com/carlosframework/platform/internal/hostclaim"
26 "github.com/carlosframework/platform/internal/hostfiles"
27 "github.com/carlosframework/platform/internal/hostguard"
28 "github.com/carlosframework/platform/internal/hostops"
29 "github.com/carlosframework/platform/internal/instancectl"
30 "github.com/carlosframework/platform/internal/lease"
31 "github.com/carlosframework/platform/internal/provision"
32 "github.com/carlosframework/platform/internal/registry"
33 "github.com/carlosframework/platform/internal/release"
34)
35
36type seedCLIConfiguration struct {
37 Actor hostbox.Actor
38 Claims, Executable string
39}
40
41func TestRootFilesSeedDelegatedOneShot(t *testing.T) {
42 runServingParkPeerNamespace(t, false, false, "seed-cli")
43}
44
45func rootFilesSeedDelegatedCLI(t *testing.T, binary string) {
46 for _, p := range []string{"/data", "/var/lib/carlos-claims", "/run/carlos-control", "/run/carlos", "/opt/carlos"} {
47 if err := os.Mkdir(p, 0755); err != nil {
48 t.Fatal("existing CLI fixture", p, err)
49 }
50 t.Cleanup(func() { os.RemoveAll(p) })
51 }
52 var captured *servingPreparation
53 var executor *filesSeedExecutor
54 capture := filesSeedLiveWitness{seed: func(_ *testing.T, p *servingPreparation, e *filesSeedExecutor) { captured, executor = p, e }}
55 runFilesSeedPreparationWithLive(t, "real-seed", "same-controller", []bool{true}, []filesSeedLiveWitness{capture})
56 f := captured.f
57 f.actor.ControlBucket = "deployment"
58 cfg := seedCLIConfiguration{Actor: f.actor, Claims: "/data/cli-claims", Executable: "/opt/carlos/versions/fixture/app"}
59 claims := release.NewDirStore(cfg.Claims)
60 keys, err := f.f.claims.List(t.Context(), "")
61 if err != nil {
62 t.Fatal(err)
63 }
64 for _, key := range keys {
65 b, _, err := f.f.claims.Get(t.Context(), key)
66 if err != nil {
67 t.Fatal(err)
68 }
69 if _, err := claims.Put(t.Context(), key, b, ""); err != nil {
70 t.Fatal(err)
71 }
72 }
73 f.f.claims, f.f.evidence = claims, claims
74 f.f.guard.Claims = hostguard.StoreReader{Store: claims}
75 f.f.guard.LockDir = ""
76 f.p.root.Stop.Coordinator.Claims = hostguard.StoreReader{Store: claims}
77 f.p.root.Stop.Coordinator.Guard = f.f.guard
78 f.p.root.Stop.Coordinator.Evidence = claims
79 f.p.root.Stop.Coordinator.Now = time.Now
80 production, err := NewProductionClaimStopHandler(f.p.root.Stop.Coordinator, f.f.journal, productionStopControllers{}, f.p.root.Roots)
81 if err != nil {
82 t.Fatal(err)
83 }
84 f.p.root.Stop = production.(ClaimStopHTTP)
85 // Initial installation/roster and the no-process kernel census are fixtures.
86 f.f.kernel.noProcess = true
87 f.p.root.Stop.Coordinator.Kernel = f.f.kernel
88 f.p.root.Stop.Coordinator.RetainNoProcess = f.p.retainLegacyNoProcess
89 reg, err := registry.Open("/data/registry.db")
90 if err != nil {
91 t.Fatal(err)
92 }
93 t.Cleanup(func() { reg.Close() })
94 row, err := executor.registry.Get(recoveryHost)
95 if err != nil || row == nil {
96 t.Fatal(err)
97 }
98 owned, err := f.p.root.Reconcile.Registry.Get(recoveryHost)
99 if err != nil || owned == nil || !owned.ClaimOwnership.Valid() {
100 t.Fatal("missing root-owned fixture route", err)
101 }
102 row.ClaimOwnership = owned.ClaimOwnership
103 if err := reg.Upsert(*row); err != nil {
104 t.Fatal(err)
105 }
106 // Production root and agent observe the same ownership transfer.
107 f.p.root.Reconcile.Registry = reg
108 h := servingTransportRoot(t, f)
109 containmentRoot := claimContainmentAgentRoot
110 capability := newLaunchFixture(t, hostclaim.Op{Kind: hostclaim.KindServing, Exec: f.actor.Box}).root
111 // The capability fixture assumes system.slice; this installed edge uses carlos.slice.
112 claimContainmentAgentRoot = containmentRoot
113 capability.Authenticate = h.Stop.Authenticate
114 capability.SeedController = verifyFilesSeedDelegation
115 listener, err := net.Listen("unix", controlsock.Path)
116 if err != nil {
117 t.Fatal(err)
118 }
119 if err := os.Chmod(controlsock.Path, 0666); err != nil {
120 t.Fatal(err)
121 }
122 server := &instancectl.Server{ClaimStop: ClaimRootWithLaunch{Root: h, Launch: capability}, PeerOK: func(net.Conn) error { return nil }, Logf: t.Logf}
123 ctx, cancel := context.WithCancel(context.Background())
124 done := make(chan error, 1)
125 go func() { done <- server.Serve(ctx, listener) }()
126 t.Cleanup(func() {
127 cancel()
128 if err := <-done; err != nil {
129 t.Error(err)
130 }
131 })
132 if err := os.MkdirAll(filepath.Dir(cfg.Executable), 0755); err != nil {
133 t.Fatal(err)
134 }
135 image, err := os.ReadFile(binary)
136 if err != nil {
137 t.Fatal(err)
138 }
139 if err := os.WriteFile(cfg.Executable, image, 0755); err != nil {
140 t.Fatal(err)
141 }
142 raw, _ := json.Marshal(cfg)
143 if err := os.WriteFile("/data/seed-cli.json", raw, 0600); err != nil {
144 t.Fatal(err)
145 }
146 account, err := user.Lookup("carlos")
147 if err != nil {
148 t.Fatal(err)
149 }
150 uid, _ := strconv.Atoi(account.Uid)
151 gid, _ := strconv.Atoi(account.Gid)
152 seedCLISharedLocks(t, "/run/carlos/hostguard", uid, gid)
153 if err := filepath.Walk("/data", func(path string, info os.FileInfo, err error) error {
154 if err != nil {
155 return err
156 }
157 return os.Chown(path, uid, gid)
158 }); err != nil {
159 t.Fatal(err)
160 }
161 raw, err = unitsFS.ReadFile("units/carlos-edge.service")
162 if err != nil {
163 t.Fatal(err)
164 }
165 var lines []string
166 for _, line := range strings.Split(string(raw), "\n") {
167 if strings.HasPrefix(line, "Requires=") || strings.HasPrefix(line, "After=") || strings.HasPrefix(line, "Wants=") || strings.HasPrefix(line, "ExecStart=") || strings.HasPrefix(line, "ExecReload=") || strings.HasPrefix(line, "EnvironmentFile=") || strings.HasPrefix(line, "Restart=") {
168 continue
169 }
170 lines = append(lines, line)
171 }
172 lines = append(lines, "[Service]", "Type=oneshot", "RemainAfterExit=yes", "Restart=no", "TimeoutStartSec=60s", "TimeoutStopSec=45s", "ExecStart="+binary+" -test.run=^TestRootFilesSeedCLIPayload$ -test.v -test.timeout=50s", "Environment=FILES_SEED_CLI=1 TMPDIR=/data CARLOS_HOST_CLAIM=1 CARLOS_HOST_CLAIM_ORIGINS=1 CARLOS_DATA_DIR=/data CARLOS_DEPLOYMENT_DIR=/data/cli-claims AWS_REGION=eu-west-1 AWS_DEFAULT_REGION=eu-west-1 AWS_ACCESS_KEY_ID=test-key AWS_SECRET_ACCESS_KEY=test-secret AWS_EC2_METADATA_DISABLED=true CARLOS_S3_ENDPOINT="+os.Getenv("CARLOS_S3_ENDPOINT"), "IPAddressDeny=any", "IPAddressAllow=127.0.0.1")
173 unit := "/run/systemd/system/carlos-edge.service"
174 file, err := os.OpenFile(unit, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
175 if err != nil {
176 t.Fatal(err)
177 }
178 _, err = file.WriteString(strings.Join(lines, "\n") + "\n")
179 file.Close()
180 if err != nil {
181 t.Fatal(err)
182 }
183 t.Cleanup(func() {
184 exec.Command("systemctl", "stop", "carlos-edge.service").Run()
185 os.Remove(unit)
186 exec.Command("systemctl", "daemon-reload").Run()
187 })
188 servingRunnerSystemctl(t, "daemon-reload")
189 run := func() {
190 out, err := exec.CommandContext(t.Context(), "systemctl", "start", "carlos-edge.service").CombinedOutput()
191 logs, _ := exec.Command("journalctl", "-u", "carlos-edge.service", "--no-pager").CombinedOutput()
192 t.Log(string(logs))
193 if err != nil {
194 t.Fatal("delegated CLI", err, string(out))
195 }
196 state, err := exec.Command("systemctl", "show", "carlos-edge.service", "-p", "ExecMainStatus", "-p", "Result", "-p", "SubState").Output()
197 if err != nil || !strings.Contains(string(state), "ExecMainStatus=0") || !strings.Contains(string(state), "Result=success") {
198 t.Fatal("one-shot status", string(state), err)
199 }
200 }
201 run()
202 observed := f.f.read()
203 if observed.Claim.Origins == nil || observed.Claim.Holder.WakeRetry == nil || observed.Claim.Holder.WakeRetry.AttemptsReserved != 1 || observed.Claim.Holder.WakeRetry.State != "open" {
204 t.Fatal("one-shot did not settle publication", observed)
205 }
206 files, err := executor.source.Open(t.Context(), claims, executor.assembly.Data, f.actor)
207 if err != nil {
208 t.Fatal(err)
209 }
210 if _, found, err := hostfiles.Load(t.Context(), files.(*movesFiles).Files, movesFilesPrefix(observed.Claim.Holder.Location, observed.Claim.Holder.Generation)); err != nil || !found {
211 t.Fatal("CLI did not publish", err)
212 }
213 servingRunnerSystemctl(t, "stop", "carlos-edge.service")
214 etag := observed.ETag
215 run()
216 if f.f.read().ETag != etag {
217 t.Fatal("delegated established rerun mutated claim")
218 }
219}
220
221func TestRootFilesSeedCLIPayload(t *testing.T) {
222 if os.Getenv("FILES_SEED_CLI") != "1" || os.Geteuid() == 0 {
223 t.Fatal("invoke through delegated CLI fixture")
224 }
225 body, err := os.ReadFile("/data/seed-cli.json")
226 if err != nil {
227 t.Fatal(err)
228 }
229 var cfg seedCLIConfiguration
230 if err := json.Unmarshal(body, &cfg); err != nil {
231 t.Fatal(err)
232 }
233 claimOwnerRegistration.Provider = seedCLIOwners{t: t, cfg: cfg}
234 RunSystemFiles([]string{"seed", "--host", recoveryHost})
235}
236
237type seedCLIOwners struct {
238 t *testing.T
239 cfg seedCLIConfiguration
240}
241
242func (o seedCLIOwners) OpenClaimOwners(ctx context.Context, store release.Store, r ClaimRuntimeResources) (ClaimOwnerBindings, error) {
243 _, _, defaults, _ := fix4JournalFixture(o.t)
244 owners := defaults.ClaimOwnerBindings
245 owners.Identity = handoverRecoveryIdentity(o.t, o.cfg.Actor)
246 owners.EvidenceKey = nil
247 owners.RetirementAuthority = &hostops.ProductionAuthority{Store: store, Directory: hostactors.Directory{Store: store, PublicKey: ed25519.NewKeyFromSeed(make([]byte, 32)).Public().(ed25519.PublicKey), Bucket: o.cfg.Actor.ControlBucket}, Now: time.Now}
248 data, err := lease.NewVersionedS3(ctx, map[string]string{o.cfg.Actor.Bucket: "eu-west-1"})
249 if err != nil {
250 return owners, err
251 }
252 owners.Data = data
253 owners.Local = &rev2Local{}
254 owners.Controllers = productionStopControllers{}
255 fixture := plan9D5Fixture(o.t)
256 fixture.launch.Executable = o.cfg.Executable
257 image, err := os.ReadFile(o.cfg.Executable)
258 if err != nil {
259 return owners, err
260 }
261 sum := sha256.Sum256(image)
262 fixture.launch.ExecutableSHA256 = hex.EncodeToString(sum[:])
263 fixture.launch.Args = []string{"--socket", provision.NestedSocket(recoveryHost), "--db", filepath.Join(r.DataDir, "instances", recoveryHost, recoveryHost+".db")}
264 fixture.launch.Limit = cgroup.TierDefault
265 fixture.launch.Env = []string{"CARLOS_ADMIN_TOKEN=test"}
266 owners.Serving = fixture.owners.Sources
267 files, err := productionClaimFilesSource().Open(ctx, store, data, o.cfg.Actor)
268 if err != nil {
269 return owners, err
270 }
271 owners.Serving.Files = files
272 return owners, nil
273}
274
275func TestFilesSeedCLIStopFixture(t *testing.T) {
276 var p *servingPreparation
277 capture := filesSeedLiveWitness{seed: func(_ *testing.T, got *servingPreparation, _ *filesSeedExecutor) { p = got }}
278 runFilesSeedPreparationWithLive(t, "real-seed", "same-controller", nil, []filesSeedLiveWitness{capture})
279 f := p.f
280 claims := release.NewDirStore(filepath.Join(t.TempDir(), "claims"))
281 keys, err := f.f.claims.List(t.Context(), "")
282 if err != nil {
283 t.Fatal(err)
284 }
285 for _, key := range keys {
286 body, _, err := f.f.claims.Get(t.Context(), key)
287 if err != nil {
288 t.Fatal(err)
289 }
290 if _, err := claims.Put(t.Context(), key, body, ""); err != nil {
291 t.Fatal(err)
292 }
293 }
294 f.f.claims, f.f.evidence = claims, claims
295 f.f.guard.Claims = hostguard.StoreReader{Store: claims}
296 f.f.guard.LockDir = filepath.Join(t.TempDir(), "hostguard")
297 seedCLISharedLocks(t, f.f.guard.LockDir, os.Getuid(), os.Getgid())
298 c := f.p.root.Stop.Coordinator
299 c.Claims, c.Guard, c.Evidence = hostguard.StoreReader{Store: claims}, f.f.guard, claims
300 c.Now = time.Now
301 f.f.kernel.noProcess = true
302 c.Kernel = f.f.kernel
303 f.p.root.Stop.Coordinator = c
304 c.RetainNoProcess = f.p.retainLegacyNoProcess
305 c.VerifiedRosterRevision = fixedRosterRevision(f.actor.Revision)
306 o := f.f.read()
307 h := o.Claim.Holder
308 req := StopRequest{Region: f.actor.Region, Host: o.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: f.actor.Box, Enrollment: f.actor.Enrollment, BootID: f.f.peer.BootID, ControllerID: f.f.peer.ControllerID, ExpectedClaimETag: o.ETag, ExpectedDirectoryRevision: f.actor.Binding, RosterRevision: f.actor.Revision}
309 if _, err := c.FenceAndStop(t.Context(), req); err != nil {
310 t.Fatal("CLI fixture no-process", err)
311 }
312}
313
314func seedCLISharedLocks(t *testing.T, path string, uid, gid int) {
315 t.Helper()
316 // Match bootstrap's carlos-owned, setgid shared-lock directory.
317 if err := os.Mkdir(path, 0770); err != nil {
318 t.Fatal(err)
319 }
320 if err := os.Chown(path, uid, gid); err != nil {
321 t.Fatal(err)
322 }
323 if err := os.Chmod(path, os.ModeSetgid|0770); err != nil {
324 t.Fatal(err)
325 }
326}
327