| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "context" |
| 8 | "encoding/json" |
| 9 | "errors" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "strings" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/cgroup" |
| 17 | "github.com/carlosframework/platform/internal/hostbox" |
| 18 | "github.com/carlosframework/platform/internal/hostclaim" |
| 19 | execlease "github.com/carlosframework/platform/internal/hostexec" |
| 20 | "github.com/carlosframework/platform/internal/hostguard" |
| 21 | "github.com/carlosframework/platform/internal/hostwrite" |
| 22 | "github.com/carlosframework/platform/internal/instancectl" |
| 23 | "github.com/carlosframework/platform/internal/provision" |
| 24 | "github.com/carlosframework/platform/internal/registry" |
| 25 | "golang.org/x/sys/unix" |
| 26 | ) |
| 27 | |
| 28 | // Fix round 1, I1: root reads the agent-written journal the way it opens |
| 29 | // every agent-writable path: a FIFO at the path answers at once rather |
| 30 | // than holding root under the host guard, a symlink is refused, a file |
| 31 | // past the cap is refused before it is decoded, and a cancelled context |
| 32 | // reads nothing. |
| 33 | func TestRootReaderOpensTheJournalBeneathTheDataRoot(t *testing.T) { |
| 34 | ctx := context.Background() |
| 35 | claim := &hostclaim.Claim{Host: "shop.example", Holder: &hostclaim.Holder{Incarnation: "inc", Stamp: 9, Session: 7, Generation: "P1", WriterBox: "dst"}, |
| 36 | Op: hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "op1", Phase: hostclaim.PhaseRelocatingProbing, Exec: "dst", Journal: map[string]string{"install_stamp": "9", "prepared": "P1", "probe_attempts.dst": "1"}}} |
| 37 | o := hostclaim.Observed{Host: "shop.example", Claim: claim} |
| 38 | attempt := hostclaim.WakeRetryAttempt{ID: "a", InvocationID: "b", Session: 7, Box: "dst"} |
| 39 | good := func(t *testing.T, dd string) relocationAttemptJournal { |
| 40 | t.Helper() |
| 41 | j, _ := readRelocationAttempts(dd, "shop.example", "op1") |
| 42 | j.Sessions["7"] = 1 |
| 43 | j.Starts["1"] = relocationStart{Session: 7, Generation: "P1", Stamp: 9, State: probeReserved, Attempt: attempt} |
| 44 | return j |
| 45 | } |
| 46 | t.Run("a FIFO", func(t *testing.T) { |
| 47 | dd := t.TempDir() |
| 48 | path, _ := relocationAttemptsPath(dd, "op1") |
| 49 | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { |
| 50 | t.Fatal(err) |
| 51 | } |
| 52 | if err := unix.Mkfifo(path, 0o600); err != nil { |
| 53 | t.Fatal(err) |
| 54 | } |
| 55 | done := make(chan error, 1) |
| 56 | go func() { |
| 57 | _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o) |
| 58 | done <- err |
| 59 | }() |
| 60 | select { |
| 61 | case err := <-done: |
| 62 | if !errors.Is(err, hostbox.ErrChanged) { |
| 63 | t.Fatalf("a FIFO: %v", err) |
| 64 | } |
| 65 | case <-time.After(5 * time.Second): |
| 66 | t.Fatal("root blocked on a FIFO at the journal's path") |
| 67 | } |
| 68 | }) |
| 69 | t.Run("a symlink", func(t *testing.T) { |
| 70 | dd := t.TempDir() |
| 71 | elsewhere := t.TempDir() |
| 72 | if err := good(t, elsewhere).write(elsewhere); err != nil { |
| 73 | t.Fatal(err) |
| 74 | } |
| 75 | real, _ := relocationAttemptsPath(elsewhere, "op1") |
| 76 | path, _ := relocationAttemptsPath(dd, "op1") |
| 77 | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { |
| 78 | t.Fatal(err) |
| 79 | } |
| 80 | if err := os.Symlink(real, path); err != nil { |
| 81 | t.Fatal(err) |
| 82 | } |
| 83 | if _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); !errors.Is(err, hostbox.ErrChanged) { |
| 84 | t.Fatalf("a symlink to a sound journal was followed: %v", err) |
| 85 | } |
| 86 | // The same journal in place reads. |
| 87 | if err := os.Remove(path); err != nil { |
| 88 | t.Fatal(err) |
| 89 | } |
| 90 | if err := good(t, dd).write(dd); err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | if got, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); err != nil || got != attempt { |
| 94 | t.Fatalf("the journal in place: %+v %v", got, err) |
| 95 | } |
| 96 | }) |
| 97 | t.Run("a symlinked directory", func(t *testing.T) { |
| 98 | dd := t.TempDir() |
| 99 | elsewhere := t.TempDir() |
| 100 | if err := good(t, elsewhere).write(elsewhere); err != nil { |
| 101 | t.Fatal(err) |
| 102 | } |
| 103 | if err := os.MkdirAll(filepath.Join(dd, "relocate"), 0o700); err != nil { |
| 104 | t.Fatal(err) |
| 105 | } |
| 106 | if err := os.Symlink(filepath.Join(elsewhere, "relocate", "ops"), filepath.Join(dd, "relocate", "ops")); err != nil { |
| 107 | t.Fatal(err) |
| 108 | } |
| 109 | if _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); !errors.Is(err, hostbox.ErrChanged) { |
| 110 | t.Fatalf("a symlinked ops directory was followed: %v", err) |
| 111 | } |
| 112 | }) |
| 113 | t.Run("past the cap", func(t *testing.T) { |
| 114 | dd := t.TempDir() |
| 115 | j := good(t, dd) |
| 116 | raw, _ := json.Marshal(j) |
| 117 | raw = append(raw, bytes.Repeat([]byte(" "), relocationAttemptsLimit+1-len(raw))...) |
| 118 | path, _ := relocationAttemptsPath(dd, "op1") |
| 119 | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { |
| 120 | t.Fatal(err) |
| 121 | } |
| 122 | if err := os.WriteFile(path, raw, 0o600); err != nil { |
| 123 | t.Fatal(err) |
| 124 | } |
| 125 | if _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); !errors.Is(err, hostbox.ErrChanged) { |
| 126 | t.Fatalf("a journal past the cap: %v", err) |
| 127 | } |
| 128 | // Trailing whitespace within the cap decodes. |
| 129 | if err := os.WriteFile(path, raw[:relocationAttemptsLimit], 0o600); err != nil { |
| 130 | t.Fatal(err) |
| 131 | } |
| 132 | if got, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); err != nil || got != attempt { |
| 133 | t.Fatalf("a journal at the cap: %+v %v", got, err) |
| 134 | } |
| 135 | }) |
| 136 | t.Run("a cancelled context", func(t *testing.T) { |
| 137 | dd := t.TempDir() |
| 138 | if err := good(t, dd).write(dd); err != nil { |
| 139 | t.Fatal(err) |
| 140 | } |
| 141 | gone, cancel := context.WithCancel(ctx) |
| 142 | cancel() |
| 143 | if _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(gone, o); !errors.Is(err, context.Canceled) { |
| 144 | t.Fatalf("a cancelled read: %v", err) |
| 145 | } |
| 146 | }) |
| 147 | } |
| 148 | |
| 149 | // Fix round 1, M4: root's authorize, with the relocation reader installed, |
| 150 | // over a journal relocationProbeAttempts.Reserve actually wrote for P's |
| 151 | // opening. The reservation the agent presents binds end to end: the |
| 152 | // attempt id as the role reservation, the invocation id, the launch-input |
| 153 | // digest, the peer's boot and controller. Another invocation, a journal |
| 154 | // rewritten under the agent, and a journal replaced by a FIFO are refused. |
| 155 | func TestRootAuthorizesAReservationTheRelocationJournalHolds(t *testing.T) { |
| 156 | ctx := t.Context() |
| 157 | dataDir := t.TempDir() |
| 158 | f := newROFixture(t, roOptions{}) |
| 159 | f.inputs.Launch = func(context.Context, registry.Route) (ClaimServingLaunch, error) { |
| 160 | return ClaimServingLaunch{Identity: hostwrite.Identity{Config: "sha256:config", Release: "sha256:release"}, Executable: "/opt/carlos/versions/v1/app", ExecutableSHA256: strings.Repeat("b", 64), Version: "v1", |
| 161 | Args: []string{"--socket", filepath.Join(provision.InstanceRunDir(roHost), "app.sock"), "--db", provision.NestedDBPath(dataDir, roHost)}, Env: []string{"CARLOS_ADMIN_TOKEN=test"}}, nil |
| 162 | } |
| 163 | in, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) |
| 164 | if err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | admitted := f.admit(f.read(), in) |
| 168 | h := admitted.Claim.Holder |
| 169 | enrollment := strings.Repeat("a", 32) |
| 170 | boot, err := execlease.BootID() |
| 171 | if err != nil { |
| 172 | t.Fatal(err) |
| 173 | } |
| 174 | start, err := claimConstructionProcessStart() |
| 175 | if err != nil { |
| 176 | t.Fatal(err) |
| 177 | } |
| 178 | peer := StopPeer{UID: 1001, PID: os.Getpid(), ProcessStart: start, BootID: boot, ControllerID: "controller-1"} |
| 179 | target := hostguard.FenceTarget{Region: "ie", Host: roHost, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Op: roOp, Phase: hostclaim.PhaseRelocatingProbing, Box: "dst", Enrollment: enrollment, BootID: peer.BootID, ControllerID: peer.ControllerID, ClaimETag: admitted.ETag, DirectoryRevision: "7"} |
| 180 | |
| 181 | // The agent's side: the journal Reserve writes for the admitted session. |
| 182 | attempts := &relocationProbeAttempts{Claims: f.claims, DataDir: dataDir, Box: "dst", NewID: f.inputs.NewID, Now: f.inputs.Now} |
| 183 | s := OperationStart{Admitted: admitted, Policy: roProbe, Inputs: in, Owner: target} |
| 184 | if err := attempts.Reserve(ctx, s, 0); err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | a := attempts.CurrentAttempt(s) |
| 188 | if a.ID == "" { |
| 189 | t.Fatal("no reservation") |
| 190 | } |
| 191 | |
| 192 | // Root's side: its own providers, the reader over the same data root. |
| 193 | exe := hostbox.LaunchFile{Device: 5, Inode: 6} |
| 194 | kernel := &launchKernel{path: in.Executable, want: exe, digest: in.ExecutableSHA256} |
| 195 | root := &ClaimLaunchRoot{ValidateExecutable: func(string, string, string) error { return nil }, Local: &rev2Local{}, Clock: &rev2Clock{wall: time.Now(), boot: 100}, DataRoot: dataDir, ExecLockDir: t.TempDir(), |
| 196 | Attempts: map[string]claimLaunchAttemptReader{hostclaim.KindRelocating: relocationAttemptReader{DataDir: dataDir}}, |
| 197 | Guard: hostguard.Guard{DataDir: dataDir, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: f.claims}}, |
| 198 | Actor: func(context.Context) (hostbox.Actor, error) { |
| 199 | return hostbox.Actor{Box: "dst", Enrollment: enrollment, Revision: 7, Binding: "7", Region: "ie", Eligible: true}, nil |
| 200 | }, |
| 201 | Kernel: kernel} |
| 202 | configureLaunchParticipation(t, &launchFixture{root: root, peer: peer, target: target, claims: &launchClaims{obs: admitted}}) |
| 203 | spec := &hostbox.LaunchSpec{Identity: in.Identity, Executable: in.Executable, Digest: in.ExecutableSHA256, Args: in.Args, Env: in.Env} |
| 204 | reservation := hostbox.LaunchReservation{ID: a.ID + ":main", Role: "main", InvocationID: a.InvocationID, Inputs: a.LaunchInputs, Target: target, Group: hostbox.LaunchGroup{RelativePath: instancectl.ClaimUnitControlGroup(roHost)}, Executable: exe} |
| 205 | q := hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchStart, Spec: spec, Reservation: &reservation, Executable: in.Executable, MemoryMax: cgroup.TierDefault.MemoryMax} |
| 206 | if err := root.authorize(ctx, peer, q, admitted.Claim); err != nil { |
| 207 | t.Fatalf("the reservation the journal holds: %v", err) |
| 208 | } |
| 209 | other := reservation |
| 210 | other.InvocationID = "another" |
| 211 | q.Reservation = &other |
| 212 | if err := root.authorize(ctx, peer, q, admitted.Claim); !errors.Is(err, hostbox.ErrChanged) { |
| 213 | t.Fatalf("another invocation: %v", err) |
| 214 | } |
| 215 | q.Reservation = &reservation |
| 216 | // The agent rewrites its journal: root's reservation no longer matches. |
| 217 | j, err := readRelocationAttempts(dataDir, roHost, roOp) |
| 218 | if err != nil { |
| 219 | t.Fatal(err) |
| 220 | } |
| 221 | st := j.Starts["1"] |
| 222 | st.Attempt.InvocationID = "rewritten" |
| 223 | j.Starts["1"] = st |
| 224 | if err := j.write(dataDir); err != nil { |
| 225 | t.Fatal(err) |
| 226 | } |
| 227 | if err := root.authorize(ctx, peer, q, admitted.Claim); !errors.Is(err, hostbox.ErrChanged) { |
| 228 | t.Fatalf("a rewritten journal: %v", err) |
| 229 | } |
| 230 | // A FIFO in the journal's place refuses at once. |
| 231 | path, _ := relocationAttemptsPath(dataDir, roOp) |
| 232 | if err := os.Remove(path); err != nil { |
| 233 | t.Fatal(err) |
| 234 | } |
| 235 | if err := unix.Mkfifo(path, 0o600); err != nil { |
| 236 | t.Fatal(err) |
| 237 | } |
| 238 | done := make(chan error, 1) |
| 239 | go func() { done <- root.authorize(ctx, peer, q, admitted.Claim) }() |
| 240 | select { |
| 241 | case err := <-done: |
| 242 | if !errors.Is(err, hostbox.ErrChanged) { |
| 243 | t.Fatalf("a FIFO: %v", err) |
| 244 | } |
| 245 | case <-time.After(5 * time.Second): |
| 246 | t.Fatal("root blocked on a FIFO in the journal's place") |
| 247 | } |
| 248 | } |
| 249 | |
amadan