| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "errors" |
| 6 | "os" |
| 7 | "os/user" |
| 8 | "path/filepath" |
| 9 | "strconv" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/hostactors" |
| 14 | "github.com/carlosframework/platform/internal/hostfloor" |
| 15 | "github.com/carlosframework/platform/internal/release" |
| 16 | ) |
| 17 | |
| 18 | // plan9aNewBody is the target build's executable in the M1-4 fixture. |
| 19 | var plan9aNewBody = []byte("#!/bin/sh\necho carlos new-build\n") |
| 20 | |
| 21 | // plan9aTargetFixture is guardRecoveryFixture with a reservation whose |
| 22 | // target is not the image on disk: current and recovery are the old build, |
| 23 | // the target is new-build, which has its own signed identity. |
| 24 | func plan9aTargetFixture(t *testing.T) (*applyFixture, string) { |
| 25 | t.Helper() |
| 26 | f, id := guardRecoveryFixture(t) |
| 27 | a := f.a |
| 28 | old, err := os.ReadFile(systemBinaryPath(a.optDir)) |
| 29 | if err != nil { |
| 30 | t.Fatal(err) |
| 31 | } |
| 32 | oldHash, newHash := hostfloor.Digest(old), hostfloor.Digest(plan9aNewBody) |
| 33 | sg := release.RootSigner(f.priv) |
| 34 | rel := release.SystemRelease{Version: "new-build", ReleaseTag: "v1.2.4", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{a.platform: {Key: release.SystemArtifactKey("new-build", a.platform), SHA256: newHash}}} |
| 35 | if _, err := release.PromoteSystem(t.Context(), f.store, sg, "router", "plan9a", rel, release.SystemPromoteOptions{}); err != nil { |
| 36 | t.Fatal(err) |
| 37 | } |
| 38 | identity, _, err := f.store.Get(t.Context(), release.SystemPointerKey("router", "plan9a")) |
| 39 | if err != nil { |
| 40 | t.Fatal(err) |
| 41 | } |
| 42 | if err := hostfloor.Durable(filepath.Join(a.floor.Dir, "identities", "v1.2.4.json"), identity, 0644); err != nil { |
| 43 | t.Fatal(err) |
| 44 | } |
| 45 | tx, err := a.guardTransaction() |
| 46 | if err != nil { |
| 47 | t.Fatal(err) |
| 48 | } |
| 49 | var roster hostactors.Roster |
| 50 | if err := release.VerifyRootEnvelope(f.pub, tx.Roster, &roster); err != nil { |
| 51 | t.Fatal(err) |
| 52 | } |
| 53 | h := roster.Activations[id] |
| 54 | h.Hashes = map[string][]string{tx.Actor: {oldHash, newHash, oldHash}} |
| 55 | roster.Activations[id] = h |
| 56 | if err := roster.Validate("bucket"); err != nil { |
| 57 | t.Fatal(err) |
| 58 | } |
| 59 | if tx.Roster, err = release.SignRootEnvelope(sg, roster); err != nil { |
| 60 | t.Fatal(err) |
| 61 | } |
| 62 | raw, _ := json.Marshal(tx) |
| 63 | if err := hostfloor.Durable(filepath.Join(a.floor.Dir, "activation.json"), raw, 0644); err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | return f, id |
| 67 | } |
| 68 | |
| 69 | func plan9aAbsent(t *testing.T, paths ...string) { |
| 70 | t.Helper() |
| 71 | for _, p := range paths { |
| 72 | if _, err := os.Lstat(p); !errors.Is(err, os.ErrNotExist) { |
| 73 | t.Fatalf("%s present: %v", p, err) |
| 74 | } |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | // M1-4: a transaction that never changed the box (refused or deferred |
| 79 | // before the first destructive step, or abandoned at the rename) records no |
| 80 | // completion. Root's marker and transaction go, so the box is not wedged, |
| 81 | // and the console's reservation stays held for a retry or an operator. |
| 82 | func TestPlan9AUnchangedTransactionRecordsNoCompletion(t *testing.T) { |
| 83 | t.Parallel() |
| 84 | for _, state := range []string{"prepared", "staging-failed", "late-participation", "abandoned-install"} { |
| 85 | t.Run(state, func(t *testing.T) { |
| 86 | f, id := plan9aTargetFixture(t) |
| 87 | a := f.a |
| 88 | marker := filepath.Join(a.floor.Dir, updateBegunFile) |
| 89 | record := filepath.Join(a.dataDir, hostexecDir, "pending.json") |
| 90 | if state != "prepared" { |
| 91 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 92 | t.Fatal(err) |
| 93 | } |
| 94 | } |
| 95 | switch state { |
| 96 | case "late-participation": |
| 97 | // The late re-check before the first destructive step runs |
| 98 | // under CARLOS_REGION_HA (systemparticipation.go). |
| 99 | a.regionHA = true |
| 100 | if err := os.WriteFile(record, []byte(`{"state":"pending"}`), 0600); err != nil { |
| 101 | t.Fatal(err) |
| 102 | } |
| 103 | if err := a.beginExecutableChange(t.Context(), "new-build", f.now); !errors.Is(err, errSystemDeferred) { |
| 104 | t.Fatalf("late participation = %v", err) |
| 105 | } |
| 106 | case "abandoned-install": |
| 107 | if err := a.journal(systemJournal{Phase: systemPhaseInstalling, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | a.removeJournal() |
| 111 | } |
| 112 | before, err := os.ReadFile(systemBinaryPath(a.optDir)) |
| 113 | if err != nil { |
| 114 | t.Fatal(err) |
| 115 | } |
| 116 | for range 2 { |
| 117 | if err := a.Run(t.Context(), true); err != nil { |
| 118 | t.Fatalf("unchanged transaction wedged recovery: %v", err) |
| 119 | } |
| 120 | } |
| 121 | plan9aAbsent(t, filepath.Join(a.floor.Dir, "completed.json"), marker, filepath.Join(a.floor.Dir, "activation.json")) |
| 122 | if after, _ := os.ReadFile(systemBinaryPath(a.optDir)); string(after) != string(before) { |
| 123 | t.Fatal("binary changed") |
| 124 | } |
| 125 | if len(f.calls) != 0 { |
| 126 | t.Fatal("recovery executed something", f.calls) |
| 127 | } |
| 128 | if state == "late-participation" { |
| 129 | if _, err := os.Stat(record); err != nil { |
| 130 | t.Fatal("root removed a controller record", err) |
| 131 | } |
| 132 | if err := os.Remove(record); err != nil { |
| 133 | t.Fatal(err) |
| 134 | } |
| 135 | } |
| 136 | // The retry begins the same reservation again. |
| 137 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 138 | t.Fatalf("retry blocked: %v", err) |
| 139 | } |
| 140 | }) |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | // M1-4: the target on disk is a finished transaction. |
| 145 | func TestPlan9AForwardFinishRecordsCompletion(t *testing.T) { |
| 146 | f, id := plan9aTargetFixture(t) |
| 147 | a := f.a |
| 148 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 149 | t.Fatal(err) |
| 150 | } |
| 151 | if err := a.journal(systemJournal{Phase: systemPhaseGated, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil { |
| 152 | t.Fatal(err) |
| 153 | } |
| 154 | if err := os.WriteFile(systemBinaryPath(a.optDir), plan9aNewBody, 0755); err != nil { |
| 155 | t.Fatal(err) |
| 156 | } |
| 157 | if err := a.closeGuardJournal(t.Context()); err != nil { |
| 158 | t.Fatal(err) |
| 159 | } |
| 160 | if raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json")); err != nil || string(raw) != id { |
| 161 | t.Fatalf("completion = %q, %v", raw, err) |
| 162 | } |
| 163 | plan9aAbsent(t, filepath.Join(a.floor.Dir, updateBegunFile)) |
| 164 | } |
| 165 | |
| 166 | // M1-4: a backward recovery restores the old image and still completes the |
| 167 | // transaction, because it rolled back. This pins the call-site wiring. |
| 168 | func TestPlan9ABackwardRecoveryRecordsCompletion(t *testing.T) { |
| 169 | f, id := plan9aTargetFixture(t) |
| 170 | a := f.a |
| 171 | old, err := os.ReadFile(systemBinaryPath(a.optDir)) |
| 172 | if err != nil { |
| 173 | t.Fatal(err) |
| 174 | } |
| 175 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 176 | t.Fatal(err) |
| 177 | } |
| 178 | if err := os.WriteFile(systemPrevPath(a.optDir), old, 0755); err != nil { |
| 179 | t.Fatal(err) |
| 180 | } |
| 181 | if err := os.WriteFile(systemBinaryPath(a.optDir), plan9aNewBody, 0755); err != nil { |
| 182 | t.Fatal(err) |
| 183 | } |
| 184 | // The install staged carlos.prev's plan; recovery reuses it rather than |
| 185 | // exec'ing a binary (floor.Run needs root-owned paths). |
| 186 | prevDir := filepath.Join(a.stagingDir(), "prev") |
| 187 | units := filepath.Join(prevDir, "etc", "systemd", "system") |
| 188 | if err := os.MkdirAll(units, 0755); err != nil { |
| 189 | t.Fatal(err) |
| 190 | } |
| 191 | if err := os.WriteFile(filepath.Join(units, "carlos-edge.service"), []byte("[Unit]\n"), 0644); err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | if err := os.MkdirAll(filepath.Join(prevDir, "etc", "tmpfiles.d"), 0755); err != nil { |
| 195 | t.Fatal(err) |
| 196 | } |
| 197 | if err := os.WriteFile(filepath.Join(prevDir, "etc", "tmpfiles.d", "carlos.conf"), []byte("d /run/carlos\n"), 0644); err != nil { |
| 198 | t.Fatal(err) |
| 199 | } |
| 200 | if err := writeJSONDurable(filepath.Join(prevDir, "plan.json"), bootstrapPlan{Enabled: []string{"carlos-edge.service"}}, 0600); err != nil { |
| 201 | t.Fatal(err) |
| 202 | } |
| 203 | if err := a.journal(systemJournal{Phase: systemPhaseInstalled, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil { |
| 204 | t.Fatal(err) |
| 205 | } |
| 206 | f.edgeStartedThisBoot = false |
| 207 | if err := a.Run(t.Context(), true); err != nil { |
| 208 | t.Fatal(err) |
| 209 | } |
| 210 | if got, _ := os.ReadFile(systemBinaryPath(a.optDir)); string(got) != string(old) { |
| 211 | t.Fatal("old image not restored") |
| 212 | } |
| 213 | if r := f.result(); r == nil || r.Outcome != systemOutcomeRolledBack { |
| 214 | t.Fatalf("result = %+v", r) |
| 215 | } |
| 216 | if raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json")); err != nil || string(raw) != id { |
| 217 | t.Fatalf("rolled-back transaction not completed: %q, %v", raw, err) |
| 218 | } |
| 219 | plan9aAbsent(t, filepath.Join(a.floor.Dir, updateBegunFile), a.file("journal.json")) |
| 220 | } |
| 221 | |
| 222 | // M1-4: a crash after the rollback was recorded and the journal removed, |
| 223 | // before completion, still completes on the old image. |
| 224 | func TestPlan9ARolledBackMarkerCompletesAfterCrash(t *testing.T) { |
| 225 | f, id := plan9aTargetFixture(t) |
| 226 | a := f.a |
| 227 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 228 | t.Fatal(err) |
| 229 | } |
| 230 | if err := hostfloor.MarkRolledBack(a.floor.Dir, id); err != nil { |
| 231 | t.Fatal(err) |
| 232 | } |
| 233 | if err := a.Run(t.Context(), true); err != nil { |
| 234 | t.Fatal(err) |
| 235 | } |
| 236 | if raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json")); err != nil || string(raw) != id { |
| 237 | t.Fatalf("completion = %q, %v", raw, err) |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | // M1-4: closing a rollback under someone else's marker refuses and keeps |
| 242 | // the journal; it never completes the local transaction instead. |
| 243 | func TestPlan9ARollbackCloseRefusesForeignMarker(t *testing.T) { |
| 244 | f, _ := plan9aTargetFixture(t) |
| 245 | a := f.a |
| 246 | marker := filepath.Join(a.floor.Dir, updateBegunFile) |
| 247 | if err := os.WriteFile(marker, []byte(`{"id":"other"}`), 0644); err != nil { |
| 248 | t.Fatal(err) |
| 249 | } |
| 250 | if err := a.journal(systemJournal{Phase: systemPhaseUnits, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil { |
| 251 | t.Fatal(err) |
| 252 | } |
| 253 | if err := a.closeGuardRollback(t.Context()); err == nil { |
| 254 | t.Fatal("foreign marker accepted") |
| 255 | } |
| 256 | if _, err := os.Stat(a.file("journal.json")); err != nil { |
| 257 | t.Fatal("journal discarded", err) |
| 258 | } |
| 259 | plan9aAbsent(t, filepath.Join(a.floor.Dir, "completed.json")) |
| 260 | } |
| 261 | |
| 262 | // M1-3: guard-arm refuses while root's update state is anywhere in the |
| 263 | // guard directory, including main's region marker and its temporary file. |
| 264 | func TestPlan9AGuardArmRefusesUpdateState(t *testing.T) { |
| 265 | for _, name := range []string{updateBegunFile, updateBegunFile + ".tmp", updateBegunFile + ".old"} { |
| 266 | t.Run(name, func(t *testing.T) { |
| 267 | guard, root := t.TempDir(), t.TempDir() |
| 268 | if err := os.WriteFile(filepath.Join(guard, name), []byte(`{"target":"v2"}`), 0644); err != nil { |
| 269 | t.Fatal(err) |
| 270 | } |
| 271 | err := guardArmIdle(guard, root) |
| 272 | if !errors.Is(err, hostfloor.ErrFloor) || !strings.Contains(err.Error(), name) { |
| 273 | t.Fatalf("guard-arm with %s = %v", name, err) |
| 274 | } |
| 275 | }) |
| 276 | } |
| 277 | guard, root := t.TempDir(), t.TempDir() |
| 278 | for _, name := range []string{"policy.json", "carlos", ".guard-123", "completed.json"} { |
| 279 | if err := os.WriteFile(filepath.Join(guard, name), nil, 0644); err != nil { |
| 280 | t.Fatal(err) |
| 281 | } |
| 282 | } |
| 283 | if err := guardArmIdle(guard, root); err != nil { |
| 284 | t.Fatalf("settled guard directory refused: %v", err) |
| 285 | } |
| 286 | if err := guardArmIdle(filepath.Join(guard, "absent"), root); err != nil { |
| 287 | t.Fatalf("absent guard directory refused: %v", err) |
| 288 | } |
| 289 | if err := os.WriteFile(filepath.Join(root, "journal.json"), []byte("{}"), 0600); err != nil { |
| 290 | t.Fatal(err) |
| 291 | } |
| 292 | if err := guardArmIdle(guard, root); !errors.Is(err, hostfloor.ErrFloor) { |
| 293 | t.Fatalf("open journal = %v", err) |
| 294 | } |
| 295 | } |
| 296 | |
| 297 | // M1-2: guard-arm creates the service-owned controller hint directory. |
| 298 | // Root census and launch completeness independently authorize an update. |
| 299 | func TestPlan9AGuardArmCreatesControllerDir(t *testing.T) { |
| 300 | f, id := guardRecoveryFixture(t) |
| 301 | a := f.a |
| 302 | if err := os.RemoveAll(filepath.Join(a.dataDir, hostexecDir)); err != nil { |
| 303 | t.Fatal(err) |
| 304 | } |
| 305 | verify := a.floor.Participation.VerifyLaunches |
| 306 | a.floor.Participation.VerifyLaunches = nil |
| 307 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err == nil { |
| 308 | t.Fatal("precondition: update began without root completeness") |
| 309 | } |
| 310 | me, err := user.Current() |
| 311 | if err != nil { |
| 312 | t.Skip(err) |
| 313 | } |
| 314 | original := lookupServiceUser |
| 315 | t.Cleanup(func() { lookupServiceUser = original }) |
| 316 | var asked []string |
| 317 | lookupServiceUser = func(name string) (*user.User, error) { asked = append(asked, name); return me, nil } |
| 318 | if err := guardArmControllers(a.dataDir); err != nil { |
| 319 | t.Fatal(err) |
| 320 | } |
| 321 | if len(asked) != 1 || asked[0] != serviceUser { |
| 322 | t.Fatalf("looked up %v, want %s", asked, serviceUser) |
| 323 | } |
| 324 | fi, err := os.Lstat(filepath.Join(a.dataDir, hostexecDir)) |
| 325 | if err != nil || !fi.IsDir() || fi.Mode().Perm() != 0o770 { |
| 326 | t.Fatalf("controller dir = %v, %v", fi, err) |
| 327 | } |
| 328 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err == nil { |
| 329 | t.Fatal("agent directory replaced root completeness") |
| 330 | } |
| 331 | a.floor.Participation.VerifyLaunches = verify |
| 332 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 333 | t.Fatalf("armed update still refused: %v", err) |
| 334 | } |
| 335 | lookupServiceUser = func(string) (*user.User, error) { return nil, user.UnknownUserError(serviceUser) } |
| 336 | if err := guardArmControllers(t.TempDir()); err == nil { |
| 337 | t.Fatal("arming proceeded without the service account") |
| 338 | } |
| 339 | lookupServiceUser = func(string) (*user.User, error) { |
| 340 | return &user.User{Uid: "x", Gid: strconv.Itoa(os.Getgid())}, nil |
| 341 | } |
| 342 | if err := guardArmControllers(t.TempDir()); err == nil { |
| 343 | t.Fatal("arming accepted a malformed uid") |
| 344 | } |
| 345 | } |
| 346 | |
| 347 | // I-2: an edge restart inside an apply pulls in the recover unit. On an |
| 348 | // armed box guard-exec recover hands the journal to the lock holder at once |
| 349 | // instead of waiting out systemRecoverLockWait. |
| 350 | func TestPlan9AGuardRecoverYieldsToHeldApplyLock(t *testing.T) { |
| 351 | opt := t.TempDir() |
| 352 | if held, err := recoverLockHeld(opt); err != nil || held { |
| 353 | t.Fatalf("no lock file = %v, %v", held, err) |
| 354 | } |
| 355 | plan9aAbsent(t, filepath.Join(systemRootDir(opt), systemLockFile)) |
| 356 | if err := os.MkdirAll(systemRootDir(opt), 0755); err != nil { |
| 357 | t.Fatal(err) |
| 358 | } |
| 359 | lock, err := os.OpenFile(filepath.Join(systemRootDir(opt), systemLockFile), os.O_CREATE|os.O_RDWR, 0600) |
| 360 | if err != nil { |
| 361 | t.Fatal(err) |
| 362 | } |
| 363 | defer lock.Close() |
| 364 | if held, err := recoverLockHeld(opt); err != nil || held { |
| 365 | t.Fatalf("free lock = %v, %v", held, err) |
| 366 | } |
| 367 | if err := flockExclusiveNonblock(lock); err != nil { |
| 368 | t.Skipf("flock unsupported: %v", err) |
| 369 | } |
| 370 | if held, err := recoverLockHeld(opt); err != nil || !held { |
| 371 | t.Fatalf("held lock = %v, %v", held, err) |
| 372 | } |
| 373 | lock.Close() |
| 374 | if held, err := recoverLockHeld(opt); err != nil || held { |
| 375 | t.Fatalf("released lock = %v, %v", held, err) |
| 376 | } |
| 377 | again, err := os.OpenFile(filepath.Join(systemRootDir(opt), systemLockFile), os.O_RDWR, 0) |
| 378 | if err != nil { |
| 379 | t.Fatal(err) |
| 380 | } |
| 381 | defer again.Close() |
| 382 | if err := flockExclusiveNonblock(again); err != nil { |
| 383 | t.Fatalf("probe left the lock held: %v", err) |
| 384 | } |
| 385 | } |
| 386 | |
amadan