carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux
2
3package host
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "io/fs"
10 "os"
11 "os/exec"
12 "path/filepath"
13 "reflect"
14 "sort"
15 "strings"
16 "syscall"
17 "time"
18
19 "github.com/carlosframework/platform/internal/cgroup"
20 "github.com/carlosframework/platform/internal/hostbox"
21 "github.com/carlosframework/platform/internal/hostguard"
22 "github.com/carlosframework/platform/internal/instancectl"
23 "github.com/carlosframework/platform/internal/provision"
24 "github.com/carlosframework/platform/internal/stopevidence"
25)
26
27// stopKernelRoots are the trees root observes. Production uses the real
28// /proc and /sys/fs/cgroup; the fixture tests use stand-ins.
29type stopKernelRoots struct {
30 Proc, Cgroup string
31 // Managed maps the inventory's managed-root names to directories.
32 Managed map[string]string
33 // AgentCgroup is the agent unit's delegated cgroup directory, under
34 // which the host's own containment would be.
35 AgentCgroup string
36}
37
38// stopLaunchHosts is the launch journal's host index.
39type stopLaunchHosts interface {
40 HostLaunches(context.Context, string) ([]hostbox.LaunchInventory, error)
41}
42
43func (k LinuxStopKernel) realRoots() (stopKernelRoots, error) {
44 if k.Journal == nil || k.CgroupRoot != "/sys/fs/cgroup" || k.ProcRoot != "/proc" || os.Geteuid() != 0 {
45 return stopKernelRoots{}, ErrClaimStopUnavailable
46 }
47 if err := verifyStopKernelMounts(k.CgroupRoot, k.ProcRoot); err != nil {
48 return stopKernelRoots{}, err
49 }
50 return stopKernelRoots{Proc: k.ProcRoot, Cgroup: k.CgroupRoot, Managed: k.ManagedRoots}, nil
51}
52
53// VerifyWritersStopped is root's writers-stopped observation (R3).
54func (k LinuxStopKernel) VerifyWritersStopped(ctx context.Context, t hostguard.FenceTarget, inv StopInventory, peer StopPeer, hints []stopevidence.DescriptorHint) (WritersKernelEvidence, error) {
55 roots, err := k.realRoots()
56 if err != nil {
57 return WritersKernelEvidence{}, err
58 }
59 if _, err := k.readContainedJournal(ctx, t); err != nil {
60 return WritersKernelEvidence{}, err
61 }
62 return verifyWritersAt(ctx, roots, k.Journal, t, inv, peer, hints)
63}
64
65// VerifyNoProcess is root's session-zero witness (R4).
66func (k LinuxStopKernel) VerifyNoProcess(ctx context.Context, t hostguard.FenceTarget, inv StopInventory) (StopKernelEvidence, error) {
67 roots, err := k.realRoots()
68 if err != nil {
69 return StopKernelEvidence{}, err
70 }
71 launches, ok := k.Journal.(stopLaunchHosts)
72 if !ok || k.AgentCgroup == nil {
73 return StopKernelEvidence{}, ErrClaimStopUnavailable
74 }
75 if roots.AgentCgroup, err = k.AgentCgroup(ctx); err != nil {
76 return StopKernelEvidence{}, err
77 }
78 boot, err := os.ReadFile(filepath.Join(k.ProcRoot, "sys/kernel/random/boot_id"))
79 if err != nil {
80 return StopKernelEvidence{}, err
81 }
82 if observed := strings.TrimSpace(string(boot)); observed == "" || observed != t.BootID {
83 return StopKernelEvidence{}, ErrClaimStopIdentity
84 }
85 evidence, err := verifyNoProcessAt(ctx, roots, launches, t, inv)
86 if err != nil {
87 return StopKernelEvidence{}, err
88 }
89 evidence.ObservedBootID = t.BootID
90 return evidence, nil
91}
92
93// verifyWritersAt: the session's launch journal is settled with no pending
94// role and every journaled containment is empty (scanStopCgroups, by device
95// and inode); every descriptor on the installation's database files, found
96// by root's complete scan, is a hinted replicator descriptor of the
97// authenticated agent; and the journal did not change across the scan.
98func verifyWritersAt(ctx context.Context, roots stopKernelRoots, journal hostbox.StopKernelJournalReader, t hostguard.FenceTarget, inv StopInventory, peer StopPeer, hints []stopevidence.DescriptorHint) (WritersKernelEvidence, error) {
99 if journal == nil || t.Session == 0 || peer.PID <= 0 || peer.ProcessStart == "" {
100 return WritersKernelEvidence{}, ErrClaimStopUnavailable
101 }
102 j, err := journal.ReadStopKernelJournal(ctx, t)
103 if err != nil {
104 return WritersKernelEvidence{}, err
105 }
106 if !j.WritersClosed || j.V != 1 || !sameStopSession(j.Target, t) || len(j.Cgroups) == 0 || !j.Settled || j.PendingRoles {
107 return WritersKernelEvidence{}, ErrClaimStopInventory
108 }
109 groups, err := scanStopCgroups(ctx, roots.Cgroup, j.Cgroups)
110 if err != nil {
111 return WritersKernelEvidence{}, err
112 }
113 paths, prefixes, err := rootInstallationCoverage(roots.Managed, t.Host, inv, true)
114 if err != nil {
115 return WritersKernelEvidence{}, err
116 }
117 files := make([]stopevidence.FileIdentity, 0, len(j.Files))
118 for _, f := range j.Files {
119 files = append(files, stopevidence.FileIdentity{Device: f.Device, Inode: f.Inode})
120 }
121 retained := []stopevidence.Descriptor{}
122 if len(paths)+len(prefixes)+len(files) > 0 {
123 held, err := stopevidence.HoldersWithPrefixes(ctx, roots.Proc, paths, prefixes, files)
124 if err != nil {
125 return WritersKernelEvidence{}, err
126 }
127 retained, err = stopevidence.ReplicatorOnly(ctx, roots.Proc, held, stopevidence.Agent{PID: peer.PID, Start: peer.ProcessStart}, hints)
128 if err != nil {
129 return WritersKernelEvidence{}, err
130 }
131 }
132 again, err := journal.ReadStopKernelJournal(ctx, t)
133 if err != nil || !reflect.DeepEqual(j, again) {
134 return WritersKernelEvidence{}, ErrClaimStopInventory
135 }
136 if _, err := scanStopCgroups(ctx, roots.Cgroup, j.Cgroups); err != nil {
137 return WritersKernelEvidence{}, err
138 }
139 return WritersKernelEvidence{CheckedPaths: paths, CgroupIdentities: groups, Retained: retained}, nil
140}
141
142// verifyNoProcessAt: the launch journal has no entry for this installation, the
143// host's containment under the agent's cgroup is absent or empty
144// throughout, and no process holds any of the installation's files; the
145// journal is read again after the scan.
146func verifyNoProcessAt(ctx context.Context, roots stopKernelRoots, launches stopLaunchHosts, t hostguard.FenceTarget, inv StopInventory) (StopKernelEvidence, error) {
147 if launches == nil || t.Session != 0 || !filepath.IsAbs(roots.AgentCgroup) {
148 return StopKernelEvidence{}, ErrClaimStopUnavailable
149 }
150 return verifyNoLaunchAt(ctx, roots, launches, t, inv)
151}
152
153func (k LinuxStopKernel) VerifyUnjournaled(ctx context.Context, t hostguard.FenceTarget, inv StopInventory) (StopKernelEvidence, error) {
154 roots, err := k.realRoots()
155 if err != nil {
156 return StopKernelEvidence{}, err
157 }
158 launches, ok := k.Journal.(stopLaunchHosts)
159 if !ok || k.AgentCgroup == nil || t.Session == 0 || t.FenceID == "" {
160 return StopKernelEvidence{}, ErrClaimStopUnavailable
161 }
162 if roots.AgentCgroup, err = k.AgentCgroup(ctx); err != nil {
163 return StopKernelEvidence{}, err
164 }
165 boot, err := os.ReadFile(filepath.Join(roots.Proc, "sys/kernel/random/boot_id"))
166 if err != nil || strings.TrimSpace(string(boot)) == "" {
167 return StopKernelEvidence{}, errors.Join(ErrClaimStopIdentity, err)
168 }
169 evidence, err := verifyNoLaunchAt(ctx, roots, launches, t, inv)
170 evidence.ObservedBootID = strings.TrimSpace(string(boot))
171 if err == nil {
172 evidence.CgroupIdentities = append(evidence.CgroupIdentities, "unlaunched:"+digestStop(t))
173 }
174 return evidence, err
175}
176
177func verifyNoLaunchAt(ctx context.Context, roots stopKernelRoots, launches stopLaunchHosts, t hostguard.FenceTarget, inv StopInventory) (StopKernelEvidence, error) {
178 records, err := launches.HostLaunches(ctx, t.Host)
179 if err != nil {
180 return StopKernelEvidence{}, err
181 }
182 for _, in := range records {
183 if in.Target.Host != t.Host {
184 return StopKernelEvidence{}, ErrClaimStopInventory
185 }
186 if in.Target.Incarnation == t.Incarnation && in.Target.Stamp == t.Stamp {
187 return StopKernelEvidence{}, ErrClaimStopInventory
188 }
189 }
190 containments := records
191 if t.Session != 0 {
192 containments = append(append([]hostbox.LaunchInventory(nil), records...), hostbox.LaunchInventory{Target: t})
193 }
194 groups, err := noProcessGroups(ctx, roots, t.Host, containments...)
195 if err != nil {
196 return StopKernelEvidence{}, err
197 }
198 paths, prefixes, err := rootInstallationCoverage(roots.Managed, t.Host, inv, false)
199 if err != nil {
200 return StopKernelEvidence{}, err
201 }
202 if len(paths)+len(prefixes) == 0 {
203 return StopKernelEvidence{}, ErrClaimStopInventory
204 }
205 held, err := stopevidence.HoldersWithPrefixes(ctx, roots.Proc, paths, prefixes, nil)
206 if err != nil {
207 return StopKernelEvidence{}, err
208 }
209 if len(held) != 0 {
210 return StopKernelEvidence{}, fmt.Errorf("%w: pid %d holds an installation file", ErrClaimStopIdentity, held[0].PID)
211 }
212
213 again, err := launches.HostLaunches(ctx, t.Host)
214 if err != nil || !reflect.DeepEqual(records, again) {
215 return StopKernelEvidence{}, errors.Join(ErrClaimStopInventory, err)
216 }
217 if again, err := noProcessGroups(ctx, roots, t.Host, containments...); err != nil || !reflect.DeepEqual(again, groups) {
218 return StopKernelEvidence{}, errors.Join(ErrClaimStopInventory, err)
219 }
220 journal := digestStop(records)
221 return StopKernelEvidence{InventorySHA256: digestStop(inv), LocalJournalSHA256: journal, CgroupIdentities: groups, UnitInvocationIDs: []string{}}, nil
222}
223
224// rootInstallationCoverage derives mandatory coverage from the root-owned layout.
225// Agent entries can add paths but cannot remove the host's database or container.
226// Name and prefix matches cover root's mount namespace only. Units.Gone must
227// be proved first: isolated namespaces can hide newly created or deleted names.
228func rootInstallationCoverage(managed map[string]string, host string, inv StopInventory, dataOnly bool) ([]string, []string, error) {
229 if !provision.ValidHostShape(host) || !filepath.IsAbs(managed["data"]) {
230 return nil, nil, ErrClaimStopInventory
231 }
232 dir := provision.InstanceDir(managed["data"], host)
233 rel, err := filepath.Rel(managed["data"], dir)
234 if err != nil {
235 return nil, nil, err
236 }
237 entry := StopInventoryEntry{Kind: "path", ManagedRoot: "data", RelativePath: rel, Type: syscall.S_IFDIR, Materialization: "intended"}
238 info, err := os.Lstat(dir)
239 if err == nil {
240 st, ok := info.Sys().(*syscall.Stat_t)
241 if !ok || !info.IsDir() {
242 return nil, nil, ErrClaimStopInventory
243 }
244 entry.Device, entry.Inode, entry.Materialization = uint64(st.Dev), st.Ino, "materialized"
245 entry.Identity, err = hostbox.CaptureInventoryIdentity(managed["data"], rel)
246 if err != nil {
247 return nil, nil, err
248 }
249 } else if !errors.Is(err, os.ErrNotExist) {
250 return nil, nil, err
251 }
252 inv.Entries = append(append([]StopInventoryEntry(nil), inv.Entries...), entry)
253 paths, prefixes, err := installationCoverage(managed, inv, dataOnly)
254 if err != nil {
255 return nil, nil, err
256 }
257 db := provision.NestedDBPath(managed["data"], host)
258 paths = append(paths, db, db+"-wal", db+"-shm", db+"-journal")
259 sort.Strings(paths)
260 return paths, prefixes, nil
261}
262
263// installationCoverage returns both the currently named files and the
264// inventory directory prefixes. The latter keep both root witnesses
265// effective for nested and open-but-unlinked files, including an installation
266// directory whose current listing is empty.
267func installationCoverage(managed map[string]string, inv StopInventory, dataOnly bool) ([]string, []string, error) {
268 if !inv.Complete || len(managed) == 0 {
269 return nil, nil, ErrClaimStopInventory
270 }
271 seen := map[string]bool{}
272 prefixSet := map[string]bool{}
273 dirs := 0
274 for _, e := range inv.Entries {
275 if e.Kind != "path" && e.Kind != "live-link" {
276 continue
277 }
278 if dataOnly && e.ManagedRoot != "data" {
279 continue
280 }
281 root := managed[e.ManagedRoot]
282 if !filepath.IsAbs(root) || !filepath.IsLocal(e.RelativePath) {
283 return nil, nil, ErrClaimStopInventory
284 }
285 path := filepath.Join(root, e.RelativePath)
286 if e.Type&syscall.S_IFMT != syscall.S_IFDIR {
287 seen[path], seen[path+"-wal"], seen[path+"-shm"] = true, true, true
288 continue
289 }
290 dirs++
291 prefixSet[filepath.Clean(path)] = true
292 info, err := os.Lstat(path)
293 if errors.Is(err, os.ErrNotExist) && e.Materialization != "materialized" {
294 continue
295 }
296 if err != nil {
297 return nil, nil, err
298 }
299 if !info.IsDir() || hostbox.VerifyInventoryIdentity(root, e) != nil {
300 return nil, nil, fmt.Errorf("%w: %s is not the inventory's directory", ErrClaimStopInventory, path)
301 }
302 entries, err := os.ReadDir(path)
303 if err != nil {
304 return nil, nil, err
305 }
306 for _, entry := range entries {
307 if entry.Type().IsRegular() {
308 seen[filepath.Join(path, entry.Name())] = true
309 }
310 }
311 }
312 if dataOnly && dirs == 0 {
313 return nil, nil, fmt.Errorf("%w: the inventory names no state directory", ErrClaimStopInventory)
314 }
315 out := make([]string, 0, len(seen))
316 for path := range seen {
317 out = append(out, path)
318 }
319 sort.Strings(out)
320 prefixes := make([]string, 0, len(prefixSet))
321 for prefix := range prefixSet {
322 prefixes = append(prefixes, prefix)
323 }
324 sort.Strings(prefixes)
325 return out, prefixes, nil
326}
327
328// hostContainmentEmpty checks the host's containment under the agent's
329// delegated cgroup: absent, or every cgroup in it unpopulated. It returns
330// the identity it observed.
331func hostContainmentEmpty(ctx context.Context, agent, host string) (string, error) {
332 name := cgroup.ExecName(host)
333 return emptyContainmentAt(ctx, filepath.Join(agent, name), name)
334}
335
336func emptyContainmentAt(ctx context.Context, dir, name string) (string, error) {
337 info, err := os.Lstat(dir)
338 if errors.Is(err, os.ErrNotExist) {
339 return "absent:" + name, nil
340 }
341 if err != nil {
342 return "", err
343 }
344 st, ok := info.Sys().(*syscall.Stat_t)
345 if !ok || !info.IsDir() {
346 return "", ErrClaimStopInventory
347 }
348 err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error {
349 if walkErr != nil {
350 return walkErr
351 }
352 if err := ctx.Err(); err != nil {
353 return err
354 }
355 if d.Type()&os.ModeSymlink != 0 {
356 return ErrClaimStopInventory
357 }
358 if !d.IsDir() {
359 return nil
360 }
361 events, err := os.ReadFile(filepath.Join(path, "cgroup.events"))
362 if err != nil {
363 return err
364 }
365 populated := ""
366 for _, line := range strings.Split(string(events), "\n") {
367 if f := strings.Fields(line); len(f) == 2 && f[0] == "populated" {
368 if populated != "" {
369 return ErrClaimStopInventory
370 }
371 populated = f[1]
372 }
373 }
374 if populated != "0" {
375 return fmt.Errorf("%w: %s is populated", ErrClaimStopInventory, path)
376 }
377 return nil
378 })
379 if err != nil {
380 return "", err
381 }
382 return fmt.Sprintf("empty:%s:%d:%d", name, st.Dev, st.Ino), nil
383}
384
385// agentUnitCgroup is the production AgentCgroup: the edge unit's cgroup as
386// systemd reports it. The agent's cgroup manager delegates its hosts'
387// containments below it.
388func agentUnitCgroup(ctx context.Context) (string, error) {
389 ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
390 defer cancel()
391 out, err := exec.CommandContext(ctx, "systemctl", "show", "-p", "ControlGroup", "--value", "carlos-edge.service").Output()
392 if err != nil {
393 return "", err
394 }
395 rel := strings.TrimPrefix(strings.TrimSpace(string(out)), "/")
396 if rel == "" || !filepath.IsLocal(rel) {
397 return "", ErrClaimStopUnavailable
398 }
399 return filepath.Join("/sys/fs/cgroup", rel), nil
400}
401
402// noProcessGroups includes every isolated role beneath the host slice.
403func noProcessGroups(ctx context.Context, roots stopKernelRoots, host string, records ...hostbox.LaunchInventory) ([]string, error) {
404 legacy, err := hostContainmentEmpty(ctx, roots.AgentCgroup, host)
405 if err != nil {
406 return nil, err
407 }
408 if !filepath.IsAbs(roots.Cgroup) {
409 return nil, ErrClaimStopUnavailable
410 }
411 rel := filepath.Dir(instancectl.ClaimUnitControlGroup(host))
412 isolated, err := emptyContainmentAt(ctx, filepath.Join(roots.Cgroup, rel), rel)
413 if err != nil {
414 return nil, err
415 }
416 out := []string{legacy, isolated}
417 seen := map[string]bool{}
418 for _, in := range records {
419 name := cgroup.ExecName(claimContainmentName(in.Target))
420 if seen[name] {
421 continue
422 }
423 seen[name] = true
424 observed, err := emptyContainmentAt(ctx, filepath.Join(roots.AgentCgroup, name), name)
425 if err != nil {
426 return nil, err
427 }
428 out = append(out, observed)
429 }
430 return out, nil
431}
432
433func rootStopAgentCgroup(ctx context.Context) (string, error) {
434 ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
435 defer cancel()
436 raw, err := exec.CommandContext(ctx, "systemctl", "show", "-p", "ControlGroup,Id,Slice,ActiveState", "carlos-edge.service").Output()
437 if err != nil {
438 return "", err
439 }
440 return rootStopAgentGroup(raw)
441}
442
443func rootStopAgentGroup(raw []byte) (string, error) {
444 fields := map[string]string{}
445 for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
446 key, value, ok := strings.Cut(line, "=")
447 if !ok {
448 return "", ErrClaimStopUnavailable
449 }
450 if _, exists := fields[key]; exists {
451 return "", ErrClaimStopUnavailable
452 }
453 fields[key] = value
454 }
455 if fields["Id"] != "carlos-edge.service" {
456 return "", ErrClaimStopUnavailable
457 }
458 rel := strings.TrimPrefix(fields["ControlGroup"], "/")
459 if rel == "" {
460 if fields["ActiveState"] != "inactive" && fields["ActiveState"] != "failed" {
461 return "", ErrClaimStopUnavailable
462 }
463 slice := fields["Slice"]
464 if !strings.HasSuffix(slice, ".slice") || !filepath.IsLocal(slice) || filepath.Base(slice) != slice {
465 return "", ErrClaimStopUnavailable
466 }
467 if slice != "-.slice" {
468 parts := strings.Split(strings.TrimSuffix(slice, ".slice"), "-")
469 for i, part := range parts {
470 if part == "" {
471 return "", ErrClaimStopUnavailable
472 }
473 rel = filepath.Join(rel, strings.Join(parts[:i+1], "-")+".slice")
474 }
475 }
476 rel = filepath.Join(rel, fields["Id"])
477 }
478 if !filepath.IsLocal(rel) || filepath.Clean(rel) != rel {
479 return "", ErrClaimStopUnavailable
480 }
481 return filepath.Join("/sys/fs/cgroup", rel), nil
482}
483
484func stopNeverIntended(records []hostbox.LaunchInventory, t hostguard.FenceTarget) bool {
485 found := false
486 for _, in := range records {
487 if in.Target.Host != t.Host {
488 return false
489 }
490 if in.Target.Incarnation != t.Incarnation || in.Target.Stamp != t.Stamp {
491 continue
492 }
493 if !in.WritersClosed || len(in.Groups) != 1 {
494 return false
495 }
496 found = true
497 for _, r := range in.Reservations {
498 if !hostbox.IsUnitReservation(r) || r.State != hostbox.UnitSettled || !r.NeverIntended || r.PID != 0 || r.ProcessStart != "" || r.Group.Device != 0 || r.Group.Inode != 0 || r.Group.InvocationID != "" {
499 return false
500 }
501 }
502 }
503 return found
504}
505