amadan

carlos / platform Public

region HA 2b: install-channel removes its cache entry by descriptor and leaves managed boxes alone

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude ef52de59c31ab96f93b97c10b6c6a0424ffa6dd5 parent c547f40
5 files changed, +251 −8
  • internal/host/installchannel.go +27 −5
  • internal/host/installchannel_linux.go +64 −1
  • internal/host/installchannel_other.go +2 −0
  • internal/host/installchannel_test.go +154 −0
  • internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv +4 −2
diff --git a/internal/host/installchannel.go b/internal/host/installchannel.go
index cfd7e4ccd..5e6e96ea2 100644
--- a/internal/host/installchannel.go
+++ b/internal/host/installchannel.go
@@ -160,17 +160,39 @@ func (in channelInstall) install(ctx context.Context) (channelInstalled, error)
if err != nil {
return got, err
}
- if applied != nil && systemCacheHolds(systemBinaryPath(a.optDir), art.SHA256) && systemCacheHolds(systemPrevPath(a.optDir), art.SHA256) &&
- systemCacheHolds(channelIdentityPath(a.optDir), hostfloor.Digest(wire)) {
- got.Already = true
- return got, nil
+ already := systemCacheHolds(systemBinaryPath(a.optDir), art.SHA256) && systemCacheHolds(systemPrevPath(a.optDir), art.SHA256) &&
+ systemCacheHolds(channelIdentityPath(a.optDir), hostfloor.Digest(wire))
+ if applied != nil {
+ // applied.json is the last thing a run writes, so a box with one is
+ // either this install, complete, or a box an update has since
+ // managed: its releases come through carlos system update.
+ if already {
+ got.Already = true
+ return got, nil
+ }
+ return got, fmt.Errorf("this box already runs %s, so carlos system update installs its releases", applied.Version)
+ }
+ // With no applied.json, a slot that holds anything but the artifact is a
+ // binary this command did not put there. A run cut off part way leaves
+ // only the artifact behind (carlos.prev is written first), so a retry
+ // passes. The slot's version is not read: that would run a binary
+ // nothing has verified, as root, so the refusal names the slot.
+ for _, slot := range []string{systemBinaryPath(a.optDir), systemPrevPath(a.optDir)} {
+ if _, err := os.Lstat(slot); err == nil && !systemCacheHolds(slot, art.SHA256) {
+ return got, fmt.Errorf("this box already runs %s, so carlos system update installs its releases", slot)
+ } else if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return got, fmt.Errorf("this box's install record could not be read, so nothing was installed: %w", err)
+ }
}
cache := systemCachePath(a.dataDir, p.Current.Version)
- defer os.RemoveAll(filepath.Dir(cache))
defer os.RemoveAll(a.stagingDir())
if err := in.cacheDir(p.Current.Version); err != nil {
return got, err
}
+ // Only now is the version directory ours to remove, and it is removed
+ // through the held router directory, never by a path that could pass a
+ // symlink (the service user owns the parents).
+ defer func() { _ = removeCacheEntry(a.dataDir, p.Current.Version) }()
if err := in.fetchTo(ctx, a.store, art.Key, cache, art.SHA256); err != nil {
return got, fmt.Errorf("%s could not be fetched, so nothing was installed: %w", p.Current.Version, err)
}
diff --git a/internal/host/installchannel_linux.go b/internal/host/installchannel_linux.go
index c18b692c8..62dfa974f 100644
--- a/internal/host/installchannel_linux.go
+++ b/internal/host/installchannel_linux.go
@@ -2,7 +2,13 @@
package host
-import "golang.org/x/sys/unix"
+import (
+ "errors"
+ "os"
+ "path/filepath"
+
+ "golang.org/x/sys/unix"
+)
// ownByService hands one directory, and nothing beneath it, to the service
// user and its group, as bootstrap's ownership step does
@@ -27,3 +33,60 @@ func ownDir(path string, uid, gid int) error {
defer unix.Close(fd)
return unix.Fchown(fd, uid, gid)
}
+
+// removeCacheEntry removes the agent's cache entry for version, the one
+// directory install-channel made for the fetch, and what is in it. It never
+// resolves a path name through a symlink: the router directory is opened
+// as bootstrapOpenDir opens one (no symlink anywhere on the path), the
+// version directory is opened beneath it with no symlink followed, and
+// everything goes by unlinkat on a held descriptor. The service user owns
+// the parents, so a path-based RemoveAll could be walked out of /data. An
+// entry that is not there is already removed.
+func removeCacheEntry(dataDir, version string) error {
+ parts := systemCacheParts(version)
+ router, err := bootstrapOpenDir(filepath.Join(append([]string{dataDir}, parts[:len(parts)-2]...)...))
+ if err != nil {
+ if errors.Is(err, unix.ENOENT) {
+ return nil
+ }
+ return err
+ }
+ defer unix.Close(router)
+ return removeBeneath(router, parts[len(parts)-2])
+}
+
+// removeBeneath removes name, a directory, and everything under it, by
+// descriptor. A symlink is unlinked, never entered.
+func removeBeneath(parent int, name string) error {
+ fd, err := unix.Openat2(parent, name, &unix.OpenHow{
+ Flags: unix.O_RDONLY | unix.O_DIRECTORY | unix.O_CLOEXEC | unix.O_NOFOLLOW,
+ Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_SYMLINKS,
+ })
+ if errors.Is(err, unix.ENOENT) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ dir := os.NewFile(uintptr(fd), name)
+ names, err := dir.Readdirnames(-1)
+ if err != nil {
+ dir.Close()
+ return err
+ }
+ for _, n := range names {
+ err := unix.Unlinkat(fd, n, 0)
+ if errors.Is(err, unix.EISDIR) {
+ err = removeBeneath(fd, n)
+ }
+ if err != nil && !errors.Is(err, unix.ENOENT) {
+ dir.Close()
+ return err
+ }
+ }
+ dir.Close()
+ if err := unix.Unlinkat(parent, name, unix.AT_REMOVEDIR); err != nil && !errors.Is(err, unix.ENOENT) {
+ return err
+ }
+ return nil
+}
diff --git a/internal/host/installchannel_other.go b/internal/host/installchannel_other.go
index 21faa7f6b..cd262cf1f 100644
--- a/internal/host/installchannel_other.go
+++ b/internal/host/installchannel_other.go
@@ -11,3 +11,5 @@ var errOwnNotLinux = errors.New("install-channel's ownership handoff requires Li
func ownByService(path string) error { return errOwnNotLinux }
func ownDir(path string, uid, gid int) error { return errOwnNotLinux }
+
+func removeCacheEntry(dataDir, version string) error { return errOwnNotLinux }
diff --git a/internal/host/installchannel_test.go b/internal/host/installchannel_test.go
index 2acdc66f5..787d77ac4 100644
--- a/internal/host/installchannel_test.go
+++ b/internal/host/installchannel_test.go
@@ -281,3 +281,157 @@ func TestInstallChannelLeavesAnInstalledBoxToUpdates(t *testing.T) {
t.Fatalf("installed box changed to %q", cur)
}
}
+
+// A refused run removes nothing through a symlinked parent: the cleanup of
+// the version directory is made only after the directories are made, and
+// goes by descriptor, so root cannot be walked out of /data.
+func TestInstallChannelRemovesNothingThroughASymlink(t *testing.T) {
+ f := newChannelFixture(t, "v1.2.0")
+ elsewhere := t.TempDir()
+ keep := filepath.Join(elsewhere, "router", "v1.4.0", "keep")
+ if err := os.MkdirAll(filepath.Dir(keep), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(keep, []byte("mine"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(elsewhere, filepath.Join(f.a.dataDir, "system")); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := f.in.run(context.Background()); err == nil {
+ t.Fatal("a symlinked parent installed")
+ }
+ if b, err := os.ReadFile(keep); err != nil || string(b) != "mine" {
+ t.Fatalf("a refused run removed what the symlink led to: %q %v", b, err)
+ }
+ // The removal itself, called directly: a symlinked parent, and a
+ // version directory that is itself a symlink, are neither entered nor
+ // followed.
+ if err := removeCacheEntry(f.a.dataDir, "v1.4.0"); err == nil {
+ t.Fatal("the removal went through a symlinked parent")
+ }
+ if b, err := os.ReadFile(keep); err != nil || string(b) != "mine" {
+ t.Fatalf("removal followed a symlinked parent: %q %v", b, err)
+ }
+ g := newChannelFixture(t, "v1.2.0")
+ router := filepath.Join(g.a.dataDir, "system", "router")
+ if err := os.MkdirAll(router, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ other := t.TempDir()
+ if err := os.WriteFile(filepath.Join(other, "keep"), []byte("mine"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(other, filepath.Join(router, "v1.4.0")); err != nil {
+ t.Fatal(err)
+ }
+ if err := removeCacheEntry(g.a.dataDir, "v1.4.0"); err == nil {
+ t.Fatal("the removal entered a symlinked version directory")
+ }
+ if b, err := os.ReadFile(filepath.Join(other, "keep")); err != nil || string(b) != "mine" {
+ t.Fatalf("removal followed the version directory's symlink: %q %v", b, err)
+ }
+ // And a real entry, with a nested directory and a symlink inside it, is
+ // removed whole, its parents left.
+ h := newChannelFixture(t, "v1.2.0")
+ entry := filepath.Join(h.a.dataDir, "system", "router", "v1.4.0")
+ if err := os.MkdirAll(filepath.Join(entry, "sub"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(entry, "sub", "carlos"), []byte("x"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(other, filepath.Join(entry, "link")); err != nil {
+ t.Fatal(err)
+ }
+ if err := removeCacheEntry(h.a.dataDir, "v1.4.0"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(entry); !os.IsNotExist(err) {
+ t.Fatalf("entry left: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(h.a.dataDir, "system", "router")); err != nil {
+ t.Fatalf("a parent was removed: %v", err)
+ }
+ if b, err := os.ReadFile(filepath.Join(other, "keep")); err != nil || string(b) != "mine" {
+ t.Fatalf("removal followed a symlink inside the entry: %q %v", b, err)
+ }
+ if err := removeCacheEntry(h.a.dataDir, "v1.4.0"); err != nil {
+ t.Fatalf("an entry already gone: %v", err)
+ }
+}
+
+// A box an update has managed, at the channel's head, has applied.json and no
+// channel identity: its releases come through carlos system update, and its
+// carlos.prev is not overwritten.
+func TestInstallChannelRefusesAnUpdatedBoxAtTheChannelHead(t *testing.T) {
+ f := newChannelFixture(t, "v1.2.0")
+ if err := os.WriteFile(systemBinaryPath(f.a.optDir), f.body, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ prev := []byte("#!/bin/sh\necho carlos v1.3.0\n")
+ if err := os.WriteFile(systemPrevPath(f.a.optDir), prev, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := writeJSONDurable(f.a.file("applied.json"), systemApplied{Version: "v1.4.0", Channel: "stable", At: f.now}, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ _, err := f.in.run(context.Background())
+ if err == nil || err.Error() != "this box already runs v1.4.0, so carlos system update installs its releases" {
+ t.Fatalf("updated box: %v", err)
+ }
+ if cur, p := f.slots(t); !bytes.Equal(cur, f.body) || !bytes.Equal(p, prev) || f.fetches != 0 {
+ t.Fatalf("an updated box was changed: %q %q after %d fetches", cur, p, f.fetches)
+ }
+}
+
+// A binary this command did not put in a slot, with no applied.json, is
+// refused and left where it is; a run cut off part way, which leaves only the
+// artifact behind, is completed.
+func TestInstallChannelRefusesAHandRolledBoxAndCompletesACutOffRun(t *testing.T) {
+ for _, slot := range []func(string) string{systemBinaryPath, systemPrevPath} {
+ f := newChannelFixture(t, "v1.2.0")
+ path := slot(f.a.optDir)
+ if err := os.WriteFile(path, []byte("#!/bin/sh\necho hand rolled\n"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ _, err := f.in.run(context.Background())
+ if err == nil || err.Error() != "this box already runs "+path+", so carlos system update installs its releases" {
+ t.Fatalf("hand-rolled %s: %v", path, err)
+ }
+ if b, _ := os.ReadFile(path); string(b) != "#!/bin/sh\necho hand rolled\n" || f.fetches != 0 {
+ t.Fatalf("a hand-rolled binary was replaced: %q after %d fetches", b, f.fetches)
+ }
+ if _, err := os.Stat(systemCachePath(f.a.dataDir, "v1.4.0")); !os.IsNotExist(err) {
+ t.Fatalf("a refused run left a cache entry: %v", err)
+ }
+ }
+ // Cut off after carlos.prev, or after both slots: no applied.json, and
+ // only the artifact on disk.
+ for name, slots := range map[string][]func(string) string{
+ "prev only": {systemPrevPath},
+ "both slots": {systemPrevPath, systemBinaryPath},
+ } {
+ t.Run(name, func(t *testing.T) {
+ f := newChannelFixture(t, "v1.2.0")
+ for _, s := range slots {
+ if err := os.WriteFile(s(f.a.optDir), f.body, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+ got, err := f.in.run(context.Background())
+ if err != nil || got.Already {
+ t.Fatalf("retry %+v %v", got, err)
+ }
+ if cur, prev := f.slots(t); !bytes.Equal(cur, f.body) || !bytes.Equal(prev, f.body) {
+ t.Fatalf("slots hold %q %q", cur, prev)
+ }
+ if applied, err := readSystemApplied(f.a.optDir); err != nil || applied == nil || applied.Version != "v1.4.0" {
+ t.Fatalf("applied %+v %v", applied, err)
+ }
+ if _, err := os.Stat(channelIdentityPath(f.a.optDir)); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
diff --git a/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv b/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv
index b5d5840cd..beb5191b1 100644
--- a/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv
+++ b/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv
@@ -1,10 +1,12 @@
# Plan 14 census overlay region-ha-2b (region HA spec 2, plan 2b: docs/superpowers/plans/2026-10-17-region-ha-2b-joining.md), loaded last. RULED by the Plan 14 controller 2026-10-16 as census-ruling-region-ha-2b.
# Task 2: host.env from the deployment's parameters (internal/hostenv: hostenv.go, the layout, assembly and rendering, with an allowlist of names, shared values written create-only and the group-box marker added on write; cli.go, carlos system host-env write and seed; write_linux.go, the write through directories held by descriptor, and write_other.go, its refusal off Linux; awsssm.go, the SSM adapter) and the SSM SDK in go.mod. Three new root-read units. runHostEnv: seed's --from file. Ruled authority A,J4 on conditions (no box write to the parameters; allowlisted names, safe values, atomic write, checked parent; authority-selecting values cross-checked against the roster at the agent's use). writeHostEnv and checkedDir (Linux): write's fixed path, /etc/carlos/host.env, reached only through /etc and /etc/carlos opened O_NOFOLLOW|O_DIRECTORY and checked by fstat (root's, writable by no group or other), the file written 0600 in that directory, synced, renamed and the directory synced; ruled A,J4 under census-ruling-region-ha-2b, as the checked parent runHostEnv's condition asks for. The legacy seal stands.
-# Task 3: carlos system install-channel (internal/host installchannel.go, installchannel_linux.go): a fresh box installs its signed channel release into both protected slots through verifyPointer, hostfloor.CheckImage and stageCandidate, all unchanged, and on every run hands each directory on the agent's cache path (system, system/router and the version's) to the service user through bootstrap's own descriptor-anchored open, refusing a symlinked parent. One new root-read unit, channelInstall.run, whose one read site opens root's apply lock (systemApplier's lockPath). Ruled authority F,J4. stageCandidate, accepted-legacy, is called and not changed. The legacy seal stands.
+# Task 3: carlos system install-channel (internal/host installchannel.go, installchannel_linux.go): a fresh box installs its signed channel release into both protected slots through verifyPointer, hostfloor.CheckImage and stageCandidate, all unchanged, and on every run hands each directory on the agent's cache path (system, system/router and the version's) to the service user through bootstrap's own descriptor-anchored open, refusing a symlinked parent. One new root-read unit, channelInstall.run, whose one read site opens root's apply lock (systemApplier's lockPath). Ruled authority F,J4. stageCandidate, accepted-legacy, is called and not changed. The legacy seal stands. Fix round 1: two more root-read units, channelInstall.install (an lstat of each protected slot, to refuse a slot holding a binary this command did not put there) and removeBeneath (Linux; removes the agent's cache entry only by openat2 and unlinkat on descriptors anchored at the router directory, opened with no symlink followed, never by path name). Both ruled F,J4 on the same footing as channelInstall.run.
@coverage . be7c96bb2ff00b5b018b436f976f0d33bbcc8765c17e357baa6cd0290f59f42d
@coverage internal/hostenv 45d9d4b90a5d441020f5ba4573821ca92701246c80b32d6f1823665fb0f65208
internal/hostenv/cli.go runHostEnv authority A,J4 rev3=census-ruling-region-ha-2b;basis=carlos system host-env: write runs as root at a box's start-up and writes /etc/carlos/host.env, 0600, whole (temporary file and rename), from allowlisted parameters with safe values only, refusing a missing required value; its one read site is seed's --from file, an operator's host.env whose values go only to the parameter store; write's path is fixed and reached only through writeHostEnv file:ReadFile@1500 df4470176b229b311927ff0ac4f268d354c3a2357cac4300ac0992d2ebb90899
internal/hostenv/write_linux.go checkedDir authority A,J4 rev3=census-ruling-region-ha-2b;basis=carlos system host-env write (root, Linux): opens one directory under one already held, O_NOFOLLOW|O_DIRECTORY, creating it 0755 first only when asked and missing, and refuses it unless fstat shows a directory owned by this process's user (root, as write requires) that no group or other can write file:Fstat@576,file:Openat@168,file:Openat@372 012257ac0866f3385044785d8abae9c31e61134928c394e22a2aeaef39ec9f3c
internal/hostenv/write_linux.go writeHostEnv authority A,J4 rev3=census-ruling-region-ha-2b;basis=carlos system host-env write (root, Linux): writes /etc/carlos/host.env only through /etc and /etc/carlos held by descriptor and checked by checkedDir; the new file is created 0600, O_EXCL|O_NOFOLLOW, in that directory, synced, renamed over host.env within it, and the directory synced; its read sites open the root and the temporary file file:Open@68,file:Openat@509 1245fc0f819ba73fb45545ffdcf92d5d4941802175e73bac209ab4d4238adff1
-@coverage internal/host 64ee4f561f2a28d584a6f7c82e068e57db4822f6cbb130116284bca583d2d390
+@coverage internal/host ad94a5be72fa655670bf48a74a80acaad2668df5169a9fcbc9bd2fda9c6785af
internal/host/installchannel.go channelInstall.run authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, at a fresh box's start-up): its one read site opens root's apply lock; the install goes through verifyPointer, hostfloor.CheckImage and stageCandidate into both protected slots, and refuses a box whose guard is armed or that already runs another release file:OpenFile@196 f0cdb6dd28e83da282c3b0d7bc2012dcd80db181a8cecbf1a2db6bcc41d1d5e3
+internal/host/installchannel.go channelInstall.install authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, at a fresh box's start-up): its one read site is an lstat of each protected slot, so that a slot holding a binary this command did not put there, on a box with no install record, is refused and left as it is; the slot's own path names it and nothing is executed file:Lstat@2787 17cbca29dcc9bf5df5b901f71aafbe68bde808c0b675bf857b864ebfe1b29058
+internal/host/installchannel_linux.go removeBeneath authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, Linux): removes the agent's cache entry for the version, after a run that made it, only through the router directory held by descriptor (opened with no symlink anywhere on its path) and its version directory opened beneath it with RESOLVE_BENEATH|RESOLVE_NO_SYMLINKS, unlinking by descriptor; a symlink is unlinked, never entered, so nothing outside /data/system/router/<version> is touched file:Openat2@64,file:Readdirnames@386 146d530b92c37b22a70758456529e9ad23bf93abd1269fa234d736e57ef4d2d6