| 1 | package instancectl |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "net/http" |
| 8 | "net/netip" |
| 9 | "os" |
| 10 | "path/filepath" |
| 11 | "strings" |
| 12 | "testing" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/provision" |
| 15 | ) |
| 16 | |
| 17 | // ensureExe lays out exactly the two links the templates bind (#214 item |
| 18 | // 2): app at the host's live link, live/<host> at ../app, nothing else in |
| 19 | // live/, and a second call repoints rather than failing. |
| 20 | func TestEnsureExeLaysOutOneBuild(t *testing.T) { |
| 21 | root := filepath.Join(t.TempDir(), "exe") |
| 22 | if err := ensureExe(root, "h.example.com", "/opt/carlos/live/h.example.com", "h.example.com"); err != nil { |
| 23 | t.Fatal(err) |
| 24 | } |
| 25 | if err := os.WriteFile(filepath.Join(root, "h.example.com", "live", "stray"), nil, 0o644); err != nil { |
| 26 | t.Fatal(err) |
| 27 | } |
| 28 | if err := ensureExe(root, "h.example.com", "/opt/carlos/live/h.example.com", "h.example.com"); err != nil { |
| 29 | t.Fatal(err) |
| 30 | } |
| 31 | if got, _ := os.Readlink(filepath.Join(root, "h.example.com", "app")); got != "/opt/carlos/live/h.example.com" { |
| 32 | t.Errorf("app -> %q", got) |
| 33 | } |
| 34 | if got, _ := os.Readlink(filepath.Join(root, "h.example.com", "live", "h.example.com")); got != "../app" { |
| 35 | t.Errorf("live/h.example.com -> %q", got) |
| 36 | } |
| 37 | entries, _ := os.ReadDir(filepath.Join(root, "h.example.com", "live")) |
| 38 | if len(entries) != 1 { |
| 39 | t.Errorf("live/ holds %d entries, want 1", len(entries)) |
| 40 | } |
| 41 | for _, bad := range [][2]string{{"..", "h"}, {"a/b", "h"}, {"x", "../h"}, {"", "h"}} { |
| 42 | if err := ensureExe(root, bad[0], "/t", bad[1]); err == nil { |
| 43 | t.Errorf("ensureExe(%q, %q) accepted", bad[0], bad[1]) |
| 44 | } |
| 45 | } |
| 46 | } |
| 47 | |
| 48 | // A long host's hashed identifier keys the directory; the link inside is |
| 49 | // still named for the host, the path the trampoline execs. |
| 50 | func TestStartWritesTheHostsExeBeforeStarting(t *testing.T) { |
| 51 | exe := filepath.Join(t.TempDir(), "exe") |
| 52 | ctl := &fakeCtl{} |
| 53 | _, c := newServer(t, ctl, func(s *Server) { s.ExeRoot = exe }) |
| 54 | host := "a-very-long-hostname-that-gets-hashed.bdm.oncarlos.com" |
| 55 | resp := post(t, c, "/start?host="+host+"&role=run&memory_max=536870912") |
| 56 | if resp.StatusCode != 200 { |
| 57 | t.Fatalf("status %d", resp.StatusCode) |
| 58 | } |
| 59 | iid := provision.InstanceID(host) |
| 60 | if iid == host { |
| 61 | t.Fatal("fixture host is not long enough to be hashed") |
| 62 | } |
| 63 | if got, _ := os.Readlink(filepath.Join(exe, iid, "live", host)); got != "../app" { |
| 64 | t.Errorf("live/%s -> %q (dir %s)", host, got, iid) |
| 65 | } |
| 66 | } |
| 67 | |
| 68 | func TestResolverAllowPolicy(t *testing.T) { |
| 69 | conf := []byte("nameserver 172.31.0.2\nnameserver 127.0.0.53\nnameserver 127.0.0.54\nnameserver 127.0.0.1\nnameserver ::1\n" + |
| 70 | "nameserver 169.254.169.254\nnameserver 169.254.169.253\nnameserver 169.254.1.1\nnameserver fd00:ec2::253\n" + |
| 71 | "nameserver fe80::1%eth0\nnameserver 0.0.0.0\nnameserver 172.31.0.2\noptions edns0\n") |
| 72 | allow, refused := resolverAllow(conf, boxPrefixes) |
| 73 | if got, want := strings.Join(allow, " "), "169.254.169.253/32 172.31.0.2/32 fd00:ec2::253/128"; got != want { |
| 74 | t.Errorf("allow = %q, want %q", got, want) |
| 75 | } |
| 76 | if len(refused) != 0 { |
| 77 | t.Errorf("refused = %q", refused) |
| 78 | } |
| 79 | // review F1: resolved's stubs are loopback, and an allow on them is |
| 80 | // an allow on every wildcard listener of the box. |
| 81 | for _, stub := range []string{"127.0.0.53", "127.0.0.54"} { |
| 82 | if ok, why := ResolverAllowed(netip.MustParseAddr(stub)); ok || !strings.Contains(why, "upstreams") { |
| 83 | t.Errorf("%s allowed=%v %q", stub, ok, why) |
| 84 | } |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | // Where resolved runs, its upstream list is what instances see (the |
| 89 | // template binds it over /etc/resolv.conf), so it is what gets opened; |
| 90 | // without it, the box's own file. |
| 91 | func TestAllowFollowsTheFileInstancesSee(t *testing.T) { |
| 92 | dir := t.TempDir() |
| 93 | etc := filepath.Join(dir, "resolv.conf") |
| 94 | up := filepath.Join(dir, "upstream.conf") |
| 95 | if err := os.WriteFile(etc, []byte("nameserver 127.0.0.53\n"), 0o644); err != nil { |
| 96 | t.Fatal(err) |
| 97 | } |
| 98 | ctl := &fakeCtl{} |
| 99 | s := &Server{Systemctl: ctl.run, Logf: t.Logf, ResolvConf: etc, ResolvedConf: up, LocalAddrs: boxAddrs} |
| 100 | if _, err := s.allowResolvers(context.Background()); err != nil { |
| 101 | t.Fatal(err) |
| 102 | } |
| 103 | if got := ctl.joined(); got != "set-property --runtime carlos-instances.slice IPAddressAllow=" { |
| 104 | t.Errorf("stub only: %q", got) |
| 105 | } |
| 106 | if err := os.WriteFile(up, []byte("nameserver 10.0.0.2\n"), 0o644); err != nil { |
| 107 | t.Fatal(err) |
| 108 | } |
| 109 | ctl2 := &fakeCtl{} |
| 110 | s.Systemctl = ctl2.run |
| 111 | if _, err := s.allowResolvers(context.Background()); err != nil { |
| 112 | t.Fatal(err) |
| 113 | } |
| 114 | if got := ctl2.joined(); got != "set-property --runtime carlos-instances.slice IPAddressAllow= IPAddressAllow=10.0.0.2/32" { |
| 115 | t.Errorf("upstreams: %q", got) |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | // The allow is reset and re-written on carlos-instances.slice before the |
| 120 | // probe unit runs (#214 item 3). |
| 121 | func TestProbeRunReopensTheResolversFirst(t *testing.T) { |
| 122 | dir := t.TempDir() |
| 123 | resolv := filepath.Join(dir, "resolv.conf") |
| 124 | if err := os.WriteFile(resolv, []byte("nameserver 10.0.0.2\n"), 0o644); err != nil { |
| 125 | t.Fatal(err) |
| 126 | } |
| 127 | ctl := &fakeCtl{} |
| 128 | s, _ := newServer(t, ctl, func(s *Server) { |
| 129 | s.ResolvConf = resolv |
| 130 | s.ExeRoot = filepath.Join(dir, "exe") |
| 131 | s.LocalAddrs = boxAddrs |
| 132 | }) |
| 133 | s.RunProbe(context.Background()) |
| 134 | calls := ctl.joined() |
| 135 | want := "set-property --runtime carlos-instances.slice IPAddressAllow= IPAddressAllow=10.0.0.2/32\nstart carlos-instance-probe.service" |
| 136 | if !strings.Contains(calls, want) { |
| 137 | t.Errorf("calls:\n%s\nwant in order:\n%s", calls, want) |
| 138 | } |
| 139 | if got, _ := os.Readlink(filepath.Join(dir, "exe", "_probe", "app")); got != SystemBinary { |
| 140 | t.Errorf("probe app -> %q", got) |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | // A box on EC2: its own VPC address, loopback as a local range, and the |
| 145 | // v6 loopback and link-local. |
| 146 | var boxPrefixes = []netip.Prefix{ |
| 147 | netip.MustParsePrefix("127.0.0.0/8"), netip.MustParsePrefix("172.31.7.19/32"), |
| 148 | netip.MustParsePrefix("::1/128"), netip.MustParsePrefix("fe80::4a1:2ff:fe3b:5c7d/128"), |
| 149 | } |
| 150 | |
| 151 | func boxAddrs() ([]netip.Prefix, error) { return boxPrefixes, nil } |
| 152 | |
| 153 | // The box's own address is never opened as a resolver, however it got |
| 154 | // into resolv.conf (#214 item 3, review F1). The control service makes |
| 155 | // this check where it writes the allow, from procfs: the probe used to |
| 156 | // make it from inside its unit, where listing interfaces takes netlink |
| 157 | // and is refused (#126). |
| 158 | func TestResolverAllowRefusesTheBoxsOwnAddress(t *testing.T) { |
| 159 | conf := []byte("nameserver 172.31.7.19\nnameserver 172.31.0.2\nnameserver 10.9.0.53\n") |
| 160 | local := append([]netip.Prefix{netip.MustParsePrefix("10.9.0.0/24")}, boxPrefixes...) |
| 161 | allow, refused := resolverAllow(conf, local) |
| 162 | if got := strings.Join(allow, " "); got != "172.31.0.2/32" { |
| 163 | t.Errorf("allow = %q", got) |
| 164 | } |
| 165 | if got := strings.Join(refused, "; "); got != "172.31.7.19 is an address of this box (172.31.7.19/32); 10.9.0.53 is an address of this box (10.9.0.0/24)" { |
| 166 | t.Errorf("refused = %q", got) |
| 167 | } |
| 168 | } |
| 169 | |
| 170 | // A refused resolver is a failing line in the report, so the box is not |
| 171 | // ready, whatever the probe says; and the allow written leaves it out. |
| 172 | func TestProbeRunFailsOnALocalResolver(t *testing.T) { |
| 173 | dir := t.TempDir() |
| 174 | resolv := filepath.Join(dir, "resolv.conf") |
| 175 | if err := os.WriteFile(resolv, []byte("nameserver 172.31.7.19\n"), 0o644); err != nil { |
| 176 | t.Fatal(err) |
| 177 | } |
| 178 | ctl := &fakeCtl{} |
| 179 | s, _ := newServer(t, ctl, func(s *Server) { s.ResolvConf = resolv; s.LocalAddrs = boxAddrs }) |
| 180 | probeOK(t, s) |
| 181 | if got := ctl.joined(); !strings.HasPrefix(got, "set-property --runtime carlos-instances.slice IPAddressAllow=\n") { |
| 182 | t.Errorf("calls:\n%s", got) |
| 183 | } |
| 184 | rep, ready := s.Ready() |
| 185 | if ready || rep.Ready { |
| 186 | t.Errorf("ready with a local resolver: %+v", rep) |
| 187 | } |
| 188 | if l := rep.Lines[len(rep.Lines)-1]; l.Check != "resolver-local" || l.Verdict != "fail" || !strings.Contains(l.Detail, "172.31.7.19 is an address of this box") { |
| 189 | t.Errorf("last line = %+v", l) |
| 190 | } |
| 191 | |
| 192 | // And a good resolver is an ok line. |
| 193 | if err := os.WriteFile(resolv, []byte("nameserver 172.31.0.2\n"), 0o644); err != nil { |
| 194 | t.Fatal(err) |
| 195 | } |
| 196 | probeOK(t, s) |
| 197 | rep, ready = s.Ready() |
| 198 | if l := rep.Lines[len(rep.Lines)-1]; !ready || l.Check != "resolver-local" || l.Verdict != "ok" { |
| 199 | t.Errorf("ready=%v report=%+v", ready, rep) |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | // When the box's addresses cannot be read, nothing is opened: the allow |
| 204 | // is reset to empty and the probe does not run. |
| 205 | func TestAllowClosesWhenTheBoxsAddressesCannotBeRead(t *testing.T) { |
| 206 | dir := t.TempDir() |
| 207 | resolv := filepath.Join(dir, "resolv.conf") |
| 208 | if err := os.WriteFile(resolv, []byte("nameserver 172.31.0.2\n"), 0o644); err != nil { |
| 209 | t.Fatal(err) |
| 210 | } |
| 211 | for name, local := range map[string]func() ([]netip.Prefix, error){ |
| 212 | "error": func() ([]netip.Prefix, error) { return nil, errors.New("no procfs") }, |
| 213 | "nil": nil, |
| 214 | } { |
| 215 | ctl := &fakeCtl{} |
| 216 | s, _ := newServer(t, ctl, func(s *Server) { s.ResolvConf = resolv; s.LocalAddrs = local }) |
| 217 | probeOK(t, s) |
| 218 | if got := ctl.joined(); got != "set-property --runtime carlos-instances.slice IPAddressAllow=" { |
| 219 | t.Errorf("%s: calls:\n%s", name, got) |
| 220 | } |
| 221 | rep, ready := s.Ready() |
| 222 | if ready || len(rep.Lines) != 1 || rep.Lines[0].Check != "resolver-allow" || !strings.Contains(rep.Lines[0].Detail, "every resolver is closed") { |
| 223 | t.Errorf("%s: ready=%v report=%+v", name, ready, rep) |
| 224 | } |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | func TestParseTheBoxsAddresses(t *testing.T) { |
| 229 | dir := t.TempDir() |
| 230 | fib := filepath.Join(dir, "fib_trie") |
| 231 | v6 := filepath.Join(dir, "if_inet6") |
| 232 | // An EC2 box: /proc/net/fib_trie (Main and Local tables) and |
| 233 | // /proc/net/if_inet6, trimmed. |
| 234 | if err := os.WriteFile(fib, []byte(`Main: |
| 235 | +-- 0.0.0.0/0 3 0 5 |
| 236 | |-- 0.0.0.0 |
| 237 | /0 universe UNICAST |
| 238 | +-- 127.0.0.0/8 2 0 2 |
| 239 | +-- 127.0.0.0/31 1 0 0 |
| 240 | |-- 127.0.0.0 |
| 241 | /8 host LOCAL |
| 242 | |-- 127.0.0.1 |
| 243 | /32 host LOCAL |
| 244 | |-- 127.255.255.255 |
| 245 | /32 link BROADCAST |
| 246 | +-- 172.31.0.0/20 2 0 2 |
| 247 | |-- 172.31.0.0 |
| 248 | /20 link UNICAST |
| 249 | |-- 172.31.7.19 |
| 250 | /32 host LOCAL |
| 251 | |-- 172.31.15.255 |
| 252 | /32 link BROADCAST |
| 253 | Local: |
| 254 | +-- 0.0.0.0/0 3 0 5 |
| 255 | |-- 172.31.7.19 |
| 256 | /32 host LOCAL |
| 257 | `), 0o644); err != nil { |
| 258 | t.Fatal(err) |
| 259 | } |
| 260 | if err := os.WriteFile(v6, []byte("00000000000000000000000000000001 01 80 10 80 lo\nfe8000000000000004a102fffe3b5c7d 02 40 20 80 ens5\n"), 0o644); err != nil { |
| 261 | t.Fatal(err) |
| 262 | } |
| 263 | got, err := localPrefixes(fib, v6) |
| 264 | if err != nil { |
| 265 | t.Fatal(err) |
| 266 | } |
| 267 | if s := fmt.Sprint(got); s != "[127.0.0.0/8 127.0.0.1/32 172.31.7.19/32 172.31.7.19/32 ::1/128 fe80::4a1:2ff:fe3b:5c7d/128]" { |
| 268 | t.Errorf("local = %s", s) |
| 269 | } |
| 270 | // The resolver, the subnet's broadcast and network addresses are not |
| 271 | // local; anything on loopback is. |
| 272 | for a, want := range map[string]bool{"172.31.0.2": false, "172.31.15.255": false, "172.31.0.0": false, "127.0.0.53": true, "172.31.7.19": true} { |
| 273 | if _, ok := isLocal(netip.MustParseAddr(a), got); ok != want { |
| 274 | t.Errorf("isLocal(%s) = %v", a, ok) |
| 275 | } |
| 276 | } |
| 277 | // No IPv6 in the kernel is no v6 address; a fib_trie with no LOCAL |
| 278 | // route is not one this parser knows, and is refused. |
| 279 | if _, err := localPrefixes(fib, filepath.Join(dir, "absent")); err != nil { |
| 280 | t.Errorf("no if_inet6: %v", err) |
| 281 | } |
| 282 | if err := os.WriteFile(fib, []byte("Main:\n +-- 0.0.0.0/0 3 0 5\n"), 0o644); err != nil { |
| 283 | t.Fatal(err) |
| 284 | } |
| 285 | if _, err := localPrefixes(fib, v6); err == nil { |
| 286 | t.Error("a fib_trie with no LOCAL route was read as no addresses") |
| 287 | } |
| 288 | if _, err := localPrefixes(filepath.Join(dir, "absent"), v6); err == nil { |
| 289 | t.Error("no fib_trie was read as no addresses") |
| 290 | } |
| 291 | } |
| 292 | |
| 293 | func TestDiskFloor(t *testing.T) { |
| 294 | if f := DiskFloor(8 << 30); f != 1<<30 { |
| 295 | t.Errorf("8 GiB: %d", f) |
| 296 | } |
| 297 | if f := DiskFloor(100 << 30); f != 10<<30 { |
| 298 | t.Errorf("100 GiB: %d", f) |
| 299 | } |
| 300 | } |
| 301 | |
| 302 | // Below the floor a start and a seed are refused with 507 and no unit is |
| 303 | // touched (#214 item 1). |
| 304 | func TestStartAndSeedRefuseBelowTheDiskFloor(t *testing.T) { |
| 305 | ctl := &fakeCtl{} |
| 306 | _, c := newServer(t, ctl, func(s *Server) { |
| 307 | s.Statfs = func(string) (uint64, uint64, error) { return 900 << 20, 8 << 30, nil } |
| 308 | }) |
| 309 | for _, path := range []string{"/start?host=h.example.com&role=run&memory_max=536870912", "/seed?host=h.example.com&memory_max=536870912"} { |
| 310 | resp := post(t, c, path) |
| 311 | if resp.StatusCode != http.StatusInsufficientStorage { |
| 312 | t.Errorf("%s: status %d", path, resp.StatusCode) |
| 313 | } |
| 314 | } |
| 315 | if calls := ctl.joined(); calls != "" { |
| 316 | t.Errorf("systemctl called: %s", calls) |
| 317 | } |
| 318 | } |
| 319 | |
| 320 | // The guard warns and never stops (review F2): usage cannot be attributed |
| 321 | // reliably from here, and a stop from here would bypass the activator's |
| 322 | // fence, so a full disk refuses new starts and leaves awake instances be. |
| 323 | func TestFullDiskWarnsAndStopsNothing(t *testing.T) { |
| 324 | ctl := &fakeCtl{} |
| 325 | var logged []string |
| 326 | s := &Server{Systemctl: ctl.run, DataRoot: t.TempDir(), |
| 327 | Logf: func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }, |
| 328 | Statfs: func(string) (uint64, uint64, error) { return 100 << 20, 8 << 30, nil }} |
| 329 | if !s.warnIfFull() { |
| 330 | t.Fatal("no warning below the floor") |
| 331 | } |
| 332 | if len(logged) != 1 || !strings.Contains(logged[0], "refusing every instance start") { |
| 333 | t.Errorf("logged %q", logged) |
| 334 | } |
| 335 | if calls := ctl.joined(); calls != "" { |
| 336 | t.Errorf("systemctl called: %s", calls) |
| 337 | } |
| 338 | s.Statfs = func(string) (uint64, uint64, error) { return 4 << 30, 8 << 30, nil } |
| 339 | if s.warnIfFull() { |
| 340 | t.Error("warned with room") |
| 341 | } |
| 342 | } |
| 343 | |
amadan