carlos / platform Public

Download this file

1	package instanceexec
2
3	import (
4	"errors"
5	"fmt"
6	"net"
7	"net/netip"
8	"os"
9	"path/filepath"
10	"strings"
11	"syscall"
12	"time"
13
14	"github.com/carlosframework/platform/internal/instancectl"
15	)
16
17	// What the probe unit's own resource section sets (the units test pins
18	// the unit side), read back from its cgroup by the cpu-limits and
19	// io-weight lines.
20	const (
21	probeCPUMax = "50000 100000" // CPUQuota=50%
22	probeCPUWeight = "50"
23	probeIOWeight = "default 50"
24	)
25
26	// resolverProbePort is the non-DNS port the resolver-ports line dials on
27	// each allowed resolver: ssh, the listener a box is most likely to have
28	// bound to all addresses.
29	const resolverProbePort = 22
30
31	// Addresses the egress fence must refuse (#214 item 3), one per range,
32	// none of them anything a box would bind. A send to each is answered by
33	// the cgroup filter with EPERM before a packet leaves.
34	var (
35	privateV4 = []string{"10.255.255.254", "172.31.255.254", "192.168.255.254", "100.127.255.254", "169.254.0.1", "198.18.0.1", "192.0.0.1"}
36	// v6 has no route on most boxes; ENETUNREACH there is unverified.
37	privateV6 = []string{"fd00::1", "fec0::1", "64:ff9b::a00:1"}
38	)
39
40	// hardeningChecks are the verdict lines for #214 items 1-3 and #178.
41	func hardeningChecks(e probeEnv, add func(check, verdict, detail string), pass func(check, detail string, ok bool)) {
42	// Item 2: /opt/carlos holds the platform binary and this unit's own
43	// build, and no version store.
44	o1, ok1 := exactly(e.optRoot, "app", "carlos", "live")
45	o2, ok2 := exactly(filepath.Join(e.optRoot, "live"), instancectl.ProbeDirName)
46	pass("opt-own-build", o1+o2, ok1 && ok2)
47
48	// Item 3: the box's resolvers are reachable through the fence. This
49	// is also the live-stack control the EPERM lines below rest on.
50	resolvers := map[string]bool{}
51	var allowed []netip.Addr
52	var dnsFail []string
53	if b, err := os.ReadFile(e.resolvConf); err != nil {
54	dnsFail = append(dnsFail, err.Error())
55	} else if addrs := instancectl.ParseResolvers(b); len(addrs) == 0 {
56	dnsFail = append(dnsFail, "no nameserver in "+e.resolvConf)
57	} else {
58	for _, a := range addrs {
59	resolvers[a.String()] = true
60	if ok, why := instancectl.ResolverAllowed(a); !ok {
61	dnsFail = append(dnsFail, a.String()+": "+why)
62	continue
63	}
64	allowed = append(allowed, a)
65	network := "udp4"
66	if a.Is6() {
67	network = "udp6"
68	}
69	if err := e.sendUDP(network, netip.AddrPortFrom(a, 53).String()); err != nil {
70	dnsFail = append(dnsFail, fmt.Sprintf("%s: sendto = %v", a, err))
71	}
72	}
73	}
74	pass("dns", strings.Join(dnsFail, "; "), len(dnsFail) == 0)
75
76	// Item 3: an allowed resolver opens only a resolver. The allow is per
77	// address, both ways and every port, so a non-DNS port on it must not
78	// answer. That it is not an address of this box is the control
79	// service's check (resolver-local), made where the allow is written:
80	// listing the box's addresses takes netlink, which this unit's
81	// RestrictAddressFamilies= refuses, as it does every instance's
82	// (#126, the v0.25.0 roll).
83	var portFail []string
84	for _, a := range allowed {
85	if err := e.connectTCP(netip.AddrPortFrom(a, resolverProbePort).String(), 2*time.Second); err == nil {
86	portFail = append(portFail, fmt.Sprintf("%s: port %d answered", a, resolverProbePort))
87	}
88	}
89	pass("resolver-ports", strings.Join(portFail, "; "), len(portFail) == 0)
90
91	// Item 3: loopback is closed.
92	lo := e.sendUDP("udp4", "127.0.0.1:9")
93	pass("loopback-v4", fmt.Sprintf("sendto = %v", lo), errors.Is(lo, syscall.EPERM))
94	lo6 := e.sendUDP("udp6", "[::1]:9")
95	switch {
96	case errors.Is(lo6, syscall.EPERM):
97	add("loopback-v6", "ok", "")
98	case noV6(lo6):
99	add("loopback-v6", "unverified", fmt.Sprintf("no IPv6 on this box (%v); deny not exercised", lo6))
100	default:
101	add("loopback-v6", "fail", fmt.Sprintf("sendto = %v", lo6))
102	}
103
104	// Item 3: private and cloud-internal ranges are closed, except an
105	// address the box resolves through, which is open on purpose.
106	var v4Fail []string
107	for _, a := range privateV4 {
108	if resolvers[a] {
109	continue
110	}
111	if err := e.sendUDP("udp4", net.JoinHostPort(a, "9")); !errors.Is(err, syscall.EPERM) {
112	v4Fail = append(v4Fail, fmt.Sprintf("%s: sendto = %v", a, err))
113	}
114	}
115	pass("private-v4", strings.Join(v4Fail, "; "), len(v4Fail) == 0)
116	var v6Fail, v6Unrouted []string
117	for _, a := range privateV6 {
118	if resolvers[a] {
119	continue
120	}
121	err := e.sendUDP("udp6", net.JoinHostPort(a, "9"))
122	switch {
123	case errors.Is(err, syscall.EPERM):
124	case noV6(err):
125	v6Unrouted = append(v6Unrouted, a)
126	default:
127	v6Fail = append(v6Fail, fmt.Sprintf("%s: sendto = %v", a, err))
128	}
129	}
130	switch {
131	case len(v6Fail) > 0:
132	add("private-v6", "fail", strings.Join(v6Fail, "; "))
133	case len(v6Unrouted) > 0:
134	add("private-v6", "unverified", "no route to "+strings.Join(v6Unrouted, ", ")+"; deny not exercised")
135	default:
136	add("private-v6", "ok", "")
137	}
138
139	// #178: an IPC namespace of its own, and no host mqueue mount.
140	ino, ierr := e.ipcInode()
141	m, mok := exactly(e.devMqueue)
142	pass("ipc-private", fmt.Sprintf("ipc ns inode=%d err=%v %s", ino, ierr, m), ierr == nil && ino != initIPCInode && mok)
143
144	// #178: the trampoline's abstract socket scope holds on this kernel.
145	aerr := e.abstractScoped()
146	pass("abstract-unix", fmt.Sprint(aerr), aerr == nil)
147
148	// Item 1: CPU settings land on a unit under carlos-instances.slice.
149	cg, cerr := e.selfCgroup()
150	if cerr != nil {
151	add("cpu-limits", "fail", cerr.Error())
152	add("io-weight", "fail", cerr.Error())
153	return
154	}
155	cpuMax := readTrim(filepath.Join(cg, "cpu.max"))
156	cpuWeight := readTrim(filepath.Join(cg, "cpu.weight"))
157	pass("cpu-limits", fmt.Sprintf("cpu.max=%q cpu.weight=%q", cpuMax, cpuWeight), cpuMax == probeCPUMax && cpuWeight == probeCPUWeight)
158
159	// Item 1: the IO weight lands too, and says whether anything on this
160	// box enforces it.
161	io, ioErr := os.ReadFile(filepath.Join(cg, "io.weight"))
162	ioFirst, _, _ := strings.Cut(strings.TrimSpace(string(io)), "\n")
163	switch {
164	case ioErr != nil:
165	add("io-weight", "unverified", "no io.weight here ("+ioErr.Error()+"); IO is not weighted")
166	case ioFirst != probeIOWeight:
167	add("io-weight", "fail", fmt.Sprintf("io.weight=%q, want %q", ioFirst, probeIOWeight))
168	case !e.ioEnforced(cg):
169	add("io-weight", "unverified", "weights are set, but no disk here has io.cost or BFQ to enforce them")
170	default:
171	add("io-weight", "ok", "")
172	}
173	}
174
175	// noV6 is an answer that means the box has no IPv6 path for that
176	// address, so the fence was never asked.
177	func noV6(err error) bool {
178	return errors.Is(err, syscall.ENETUNREACH) || errors.Is(err, syscall.EADDRNOTAVAIL) || errors.Is(err, syscall.EAFNOSUPPORT)
179	}
180
181	func readTrim(path string) string {
182	b, err := os.ReadFile(path)
183	if err != nil {
184	return "<" + err.Error() + ">"
185	}
186	return strings.TrimSpace(string(b))
187	}
188