| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "crypto/ed25519" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 9 | "encoding/json" |
| 10 | "errors" |
| 11 | "flag" |
| 12 | "fmt" |
| 13 | "io" |
| 14 | "log" |
| 15 | "net/http" |
| 16 | "os" |
| 17 | "os/exec" |
| 18 | "path/filepath" |
| 19 | "runtime" |
| 20 | "slices" |
| 21 | "strings" |
| 22 | "time" |
| 23 | |
| 24 | "github.com/carlosframework/platform/internal/activator" |
| 25 | "github.com/carlosframework/platform/internal/cliflag" |
| 26 | "github.com/carlosframework/platform/internal/hostfloor" |
| 27 | "github.com/carlosframework/platform/internal/hostwrite" |
| 28 | "github.com/carlosframework/platform/internal/layout" |
| 29 | "github.com/carlosframework/platform/internal/moves" |
| 30 | "github.com/carlosframework/platform/internal/provision" |
| 31 | "github.com/carlosframework/platform/internal/registry" |
| 32 | "github.com/carlosframework/platform/internal/release" |
| 33 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 34 | "github.com/carlosframework/platform/internal/version" |
| 35 | ) |
| 36 | |
| 37 | // The root half of the platform self-update (spec §6). Runs as root from |
| 38 | // carlos-system-update.service (or --recover from carlos-system-recover). |
| 39 | // Every box-changing step is journaled first; every run begins with |
| 40 | // recovery; root reads ONE file under /data and treats it as a hint. |
| 41 | // |
| 42 | // The layout guard (layoutReversalReasons) also touches /data: it opens |
| 43 | // the registry, lists /data/layout, and Lstats the journals there. That |
| 44 | // grants no new privilege. Root's adopt, stage and restart passes already |
| 45 | // open the same registry, the listing and the Lstats follow no symlink |
| 46 | // and read no journal's contents, and all the guard can do with what it |
| 47 | // finds is refuse the install. |
| 48 | |
| 49 | const ( |
| 50 | systemEdgeUnit = "carlos-edge" |
| 51 | systemRelayUnit = "carlos-relay" |
| 52 | // systemControlUnit is the root instance-control service (isolated-exec |
| 53 | // design §5). It execs /opt/carlos/carlos and stays up for weeks once |
| 54 | // the first wake activated it, so every install that changes the |
| 55 | // binary restarts it (restartInstanceControl; discussion #256). |
| 56 | systemControlUnit = "carlos-instance-control.service" |
| 57 | systemLockFile = "lock" |
| 58 | systemMaxRequest = 4 << 10 |
| 59 | systemGateWait = 20 * time.Second |
| 60 | // systemClaimsGateWait is the gate's window on a box running with |
| 61 | // CARLOS_HOST_CLAIM on. There the agent finishes its whole claim |
| 62 | // startup (assembly, wiring, the inline activator and duties) before |
| 63 | // the edge's listener binds, which takes about 22 s on a small |
| 64 | // deployment and grows with the deployment's app count. A fixed 20 s |
| 65 | // rolled every update back, and its rollback too. The longer window |
| 66 | // comes with fail-fast (gateFailFast), so a build that crashes still |
| 67 | // rolls back in seconds. Claims-off boxes keep systemGateWait exactly. |
| 68 | systemClaimsGateWait = 120 * time.Second |
| 69 | systemGateSoak = 15 * time.Second |
| 70 | systemVersionWait = 30 * time.Second |
| 71 | systemBootstrapWait = 120 * time.Second |
| 72 | |
| 73 | // How long --recover waits for the apply lock, and how often it asks. |
| 74 | // See waitForLock. |
| 75 | systemRecoverLockWait = 30 * time.Second |
| 76 | systemRecoverLockPoll = 500 * time.Millisecond |
| 77 | ) |
| 78 | |
| 79 | // systemQuiesceUnits are the root units that exec /opt/carlos/carlos |
| 80 | // against the running edge. Stopped (services too — stopping a timer does |
| 81 | // not stop the pass it already started) before the rename, re-armed after |
| 82 | // the gate. |
| 83 | var systemQuiesceUnits = []string{ |
| 84 | "carlos-adopt.timer", "carlos-adopt.path", "carlos-stage.path", "carlos-restart.path", |
| 85 | "carlos-adopt.service", "carlos-config-sync.service", "carlos-stage.service", "carlos-restart.service", |
| 86 | } |
| 87 | |
| 88 | var ( |
| 89 | errSystemRefused = errors.New("refused") |
| 90 | errSystemDeferred = errors.New("deferred") |
| 91 | |
| 92 | // errSystemUnsupported is what the non-linux build's primitives return. |
| 93 | // It is declared here so Run can tell it from a lock another apply holds |
| 94 | // without knowing which build it is in. |
| 95 | errSystemUnsupported = errors.New("carlos system apply: unsupported on this platform") |
| 96 | ) |
| 97 | |
| 98 | // sysOps is the seam between the apply and the box: systemctl, exec with a |
| 99 | // deadline, the clock, sleep, the edge probe. Tests record and answer. |
| 100 | type sysOps struct { |
| 101 | systemctl func(args ...string) (string, error) |
| 102 | run func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) |
| 103 | now func() time.Time |
| 104 | sleep func(time.Duration) |
| 105 | probeEdge func() error |
| 106 | edgeStartedThisBoot func() bool |
| 107 | logf func(format string, args ...any) |
| 108 | } |
| 109 | |
| 110 | // RunSystemApply is `carlos system apply [-recover]`: the root oneshot |
| 111 | // behind carlos-system-update.service, and with -recover the boot and |
| 112 | // OnFailure= unit that puts an interrupted apply back together. |
| 113 | func RunSystemApply(args []string) { |
| 114 | fs := flag.NewFlagSet("system apply", flag.ExitOnError) |
| 115 | recover := fs.Bool("recover", false, "recover an interrupted apply (files only) and exit: the boot and OnFailure unit") |
| 116 | fs.Usage = cliflag.Usage(fs, "") |
| 117 | _ = fs.Parse(args) |
| 118 | if os.Geteuid() != 0 { |
| 119 | log.Fatal("carlos system apply is the root oneshot behind carlos-system-update.service; it is not for operators") |
| 120 | } |
| 121 | if err := registerProductionLaunchVerifier(); err != nil { |
| 122 | log.Fatal(err) |
| 123 | } |
| 124 | ctx := context.Background() |
| 125 | channel, invalidChannel := systemChannel(os.Getenv) |
| 126 | if invalidChannel != "" { |
| 127 | log.Printf("system apply: %s=%q is not a channel; this box follows %s", systemChannelEnv, invalidChannel, channel) |
| 128 | } |
| 129 | claims, gateWait := systemApplyGate(os.Getenv) |
| 130 | a := &systemApplier{ |
| 131 | rootMoves: rootMovesFloorIdentity, ops: productionSysOps(), openStore: openSystemStore, channel: channel, |
| 132 | optDir: release.OptDir(), dataDir: dataDir(), etcDir: "/etc", build: version.Build, |
| 133 | platform: runtime.GOOS + "-" + runtime.GOARCH, gateWait: gateWait, soak: systemGateSoak, |
| 134 | regionHA: regionHAEnabled(os.Getenv), claims: claims, |
| 135 | } |
| 136 | a.floorHook = installFloorHook // decided under the apply lock, in Run |
| 137 | runErr := a.runWithRelease(ctx, *recover) |
| 138 | if runErr != nil { |
| 139 | log.Fatalf("%v", runErr) |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | func (a *systemApplier) runWithRelease(ctx context.Context, recover bool) error { |
| 144 | if _, _, err := a.rootMoves.read(); err != nil { |
| 145 | return err |
| 146 | } |
| 147 | return a.participationDisk().WithParticipation(ctx, func(ctx context.Context, _ rootparticipation.Ledger) error { |
| 148 | return a.Run(ctx, recover) |
| 149 | }, func() error { return a.Run(ctx, recover) }) |
| 150 | } |
| 151 | |
| 152 | // openSystemStore is the production openStore: the deploy store and the |
| 153 | // release public key, opened at the moment the pointer is about to be read |
| 154 | // and not one step sooner. |
| 155 | // |
| 156 | // The lateness is the point. `carlos system apply --recover` runs at boot, |
| 157 | // before the network is up (its unit deliberately carries no |
| 158 | // After=network-online.target, because the whole job is to put the box's |
| 159 | // FILES back so the edge can start at all), and reads nothing from any |
| 160 | // store. Opening a bucket and parsing CARLOS_RELEASE_PUBKEY up front made |
| 161 | // recovery — the one path that must work on a broken box — depend on two |
| 162 | // things a broken box is most likely to be missing. |
| 163 | func openSystemStore(ctx context.Context) (release.Store, ed25519.PublicKey, error) { |
| 164 | store, err := openDeployStore(ctx) |
| 165 | if err != nil { |
| 166 | return nil, nil, err |
| 167 | } |
| 168 | pub, err := release.ParsePublicKey(os.Getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY") |
| 169 | if err != nil { |
| 170 | return nil, nil, err |
| 171 | } |
| 172 | return store, pub, nil |
| 173 | } |
| 174 | |
| 175 | // openSystemRegistry keeps root's SQLite access in the service-user child. |
| 176 | func openSystemRegistry(dataDir string) (*registry.Registry, error) { |
| 177 | path := filepath.Join(dataDir, "registry.db") |
| 178 | reg, err := openServiceRegistry(path) |
| 179 | if err != nil { |
| 180 | return nil, fmt.Errorf("%s: %w", path, err) |
| 181 | } |
| 182 | return reg, nil |
| 183 | } |
| 184 | |
| 185 | // productionSysOps is the real box behind the seam. |
| 186 | func productionSysOps() sysOps { |
| 187 | return sysOps{ |
| 188 | systemctl: func(args ...string) (string, error) { |
| 189 | out, err := exec.Command("systemctl", args...).CombinedOutput() |
| 190 | if err != nil { |
| 191 | return string(out), fmt.Errorf("systemctl %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(out))) |
| 192 | } |
| 193 | return string(out), nil |
| 194 | }, |
| 195 | run: func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { |
| 196 | ctx, cancel := context.WithTimeout(ctx, timeout) |
| 197 | defer cancel() |
| 198 | out, err := exec.CommandContext(ctx, name, args...).Output() |
| 199 | if err != nil { |
| 200 | // Output() keeps stderr out of the returned bytes so the |
| 201 | // caller can parse them; a command that failed is the one |
| 202 | // case where what it said matters more than what it printed. |
| 203 | var ee *exec.ExitError |
| 204 | if errors.As(err, &ee) && len(ee.Stderr) > 0 { |
| 205 | return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(ee.Stderr))) |
| 206 | } |
| 207 | return string(out), fmt.Errorf("%s %s: %w", name, strings.Join(args, " "), err) |
| 208 | } |
| 209 | return string(out), nil |
| 210 | }, |
| 211 | now: time.Now, |
| 212 | sleep: time.Sleep, |
| 213 | probeEdge: func() error { |
| 214 | // The edge's :80 listener answers on every box — ACME HTTP-01 |
| 215 | // and the redirect to https — so any HTTP answer on loopback |
| 216 | // proves the process serves. A redirect is an answer, and a box |
| 217 | // with no route yet must still pass (spec §6 step 10). |
| 218 | c := &http.Client{Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} |
| 219 | resp, err := c.Get("http://127.0.0.1/") |
| 220 | if err != nil { |
| 221 | return err |
| 222 | } |
| 223 | resp.Body.Close() |
| 224 | return nil |
| 225 | }, |
| 226 | edgeStartedThisBoot: func() bool { |
| 227 | out, err := exec.Command("systemctl", "show", "-p", "ActiveEnterTimestampMonotonic", "--value", systemEdgeUnit).Output() |
| 228 | return err == nil && strings.TrimSpace(string(out)) != "" && strings.TrimSpace(string(out)) != "0" |
| 229 | }, |
| 230 | logf: log.Printf, |
| 231 | } |
| 232 | } |
| 233 | |
| 234 | type systemApplier struct { |
| 235 | floorAlerted bool |
| 236 | rootMoves rootFloorIdentitySource |
| 237 | participation *rootparticipation.Ledger |
| 238 | floor *hostfloor.Disk |
| 239 | // floorHook, when set, decides legacy or floor-guarded state once Run |
| 240 | // holds the apply lock, so a concurrent run's marker write is never read. |
| 241 | floorHook func(*systemApplier) error |
| 242 | guardOpen func(string) (*os.File, error) |
| 243 | ops sysOps |
| 244 | // openStore is called once, by verifyPointer, and only on a run that |
| 245 | // has a request to answer. --recover never calls it: see |
| 246 | // openSystemStore for why that matters at boot. |
| 247 | openStore func(ctx context.Context) (release.Store, ed25519.PublicKey, error) |
| 248 | // openRegistry is called only when the candidate predates the instance |
| 249 | // layout, to see whether anything on the box is still nested. Nil means |
| 250 | // openSystemRegistry over dataDir. |
| 251 | openRegistry func() (*registry.Registry, error) |
| 252 | channel string |
| 253 | optDir string |
| 254 | dataDir string |
| 255 | etcDir string // "/etc" in production; a temp dir in tests |
| 256 | build string // version.Build of THIS process |
| 257 | platform string // "<goos>-<goarch>"; the caller sets it from runtime |
| 258 | gateWait, soak time.Duration |
| 259 | // regionHA is CARLOS_REGION_HA: root writes its update-begun marker and |
| 260 | // refuses an update while this box takes part in a claim operation |
| 261 | // (region HA spec 1 §5.10; systemparticipation.go). |
| 262 | regionHA bool |
| 263 | // claims is CARLOS_HOST_CLAIM, read with hostwrite.Enabled from the |
| 264 | // unit's environment (carlos-system-update.service and |
| 265 | // carlos-system-recover.service both load host.env). It changes only the |
| 266 | // gate: the claims window and fail-fast (systemGateWaitFor, |
| 267 | // gateFailFast). Off, the gate is exactly the 20 s one. |
| 268 | claims bool |
| 269 | // store is the deploy store verifyPointer opened, kept for armFloor. |
| 270 | // Nil on --recover, which reads no store. |
| 271 | store release.Store |
| 272 | // beforeSwap, when set, runs inside activationGuard after the rule has |
| 273 | // allowed and just before the step it guards: a test's schedule hook. |
| 274 | // Nil in production. |
| 275 | beforeSwap func() |
| 276 | } |
| 277 | |
| 278 | func (a *systemApplier) rootDir() string { return systemRootDir(a.optDir) } |
| 279 | func (a *systemApplier) stagingDir() string { return filepath.Join(a.rootDir(), "staging") } |
| 280 | func (a *systemApplier) lockPath() string { return filepath.Join(a.rootDir(), systemLockFile) } |
| 281 | func (a *systemApplier) file(name string) string { return filepath.Join(a.rootDir(), name) } |
| 282 | |
| 283 | // Run is the whole oneshot. It returns an error only for a failure the |
| 284 | // caller should exit non-zero on (an apply that rolled back, a rollback |
| 285 | // that failed, a broken box); refusals and no-ops are recorded in |
| 286 | // result.json and return nil. |
| 287 | // |
| 288 | // A run whose RECOVERY rolled the box back exits 0, unlike a rollback |
| 289 | // inside install: the result and the heartbeat carry the truth, the |
| 290 | // journal is closed, and re-firing OnFailure= over a finished recovery |
| 291 | // would only start it again with nothing left to do. |
| 292 | func (a *systemApplier) Run(ctx context.Context, recover bool) (retErr error) { |
| 293 | defer func() { |
| 294 | if errors.Is(retErr, context.DeadlineExceeded) { |
| 295 | if required, err := a.participationDisk().ParticipationRequired(context.WithoutCancel(ctx)); err == nil && required { |
| 296 | retErr = errors.Join(errSystemDeferred, retErr) |
| 297 | } |
| 298 | } |
| 299 | }() |
| 300 | reapStagingWrites(a.optDir, a.dataDir, a.etcDir, a.ops.logf) |
| 301 | if err := os.MkdirAll(a.rootDir(), 0o755); err != nil { |
| 302 | return err |
| 303 | } |
| 304 | lock, err := os.OpenFile(a.lockPath(), os.O_CREATE|os.O_RDWR, 0o600) |
| 305 | if err != nil { |
| 306 | return err |
| 307 | } |
| 308 | defer lock.Close() |
| 309 | if recover { |
| 310 | got, err := a.waitForLock(lock) |
| 311 | if err != nil { |
| 312 | return err |
| 313 | } |
| 314 | if !got { |
| 315 | a.ops.logf("system apply: an apply holds the lock; it recovers at its own start") |
| 316 | return nil |
| 317 | } |
| 318 | } else if err := flockExclusiveNonblock(lock); err != nil { |
| 319 | if errors.Is(err, errSystemUnsupported) { |
| 320 | a.ops.logf("system apply: %v", err) |
| 321 | return err |
| 322 | } |
| 323 | a.ops.logf("system apply: another apply is running; leaving it to finish") |
| 324 | return nil |
| 325 | } |
| 326 | if a.floorHook != nil { |
| 327 | if err := a.floorHook(a); err != nil { |
| 328 | return err |
| 329 | } |
| 330 | } |
| 331 | // Rejected floor recovery must retain staging and the journal. A |
| 332 | // settlement deferred with no journal open (errSettlementDeferred) lets |
| 333 | // --recover go on and exit 0; it is not tried again at the end. |
| 334 | deferred := false |
| 335 | if err := a.checkGuardRecovery(ctx); err != nil { |
| 336 | if !a.deferSettlement(recover, err) { |
| 337 | return err |
| 338 | } |
| 339 | deferred = true |
| 340 | } |
| 341 | defer func() { |
| 342 | active, err := a.activeFloor() |
| 343 | if err != nil { |
| 344 | retErr = errors.Join(retErr, err) |
| 345 | return |
| 346 | } |
| 347 | if !active { |
| 348 | if !deferred { |
| 349 | a.settleMarker(ctx, recover, &retErr) |
| 350 | } |
| 351 | _ = os.RemoveAll(a.stagingDir()) |
| 352 | return |
| 353 | } |
| 354 | if j, e := readSystemJournal(a.optDir); e == nil && j == nil && !deferred { |
| 355 | if err := a.checkGuardRecovery(ctx); err != nil { |
| 356 | if !a.deferSettlement(recover, err) { |
| 357 | retErr = errors.Join(retErr, err) |
| 358 | } |
| 359 | return |
| 360 | } |
| 361 | a.settleMarker(ctx, recover, &retErr) |
| 362 | _ = os.RemoveAll(a.stagingDir()) |
| 363 | } |
| 364 | }() |
| 365 | // Root passes an earlier run paused and could not start again come |
| 366 | // first: they are the box's convergence, whatever this run is for. |
| 367 | // After main's floor hook and guard recovery check (merge of claim |
| 368 | // Plan 8): a release-floor recovery it rejects keeps the box exactly as |
| 369 | // main leaves it, and the triggers wait for the run that gets past it. |
| 370 | a.rearmPending() |
| 371 | |
| 372 | // A settlement the census refused after recovery durably removed the |
| 373 | // journal is deferred like one with no journal open: --recover exits 0 |
| 374 | // and does not try it again at the end of the run. |
| 375 | recovered, err := a.recoverIfNeeded(ctx) |
| 376 | if err != nil { |
| 377 | if !a.deferSettlement(recover, err) { |
| 378 | return err |
| 379 | } |
| 380 | deferred = true |
| 381 | } |
| 382 | if recover { |
| 383 | return nil |
| 384 | } |
| 385 | if recovered { |
| 386 | // The recovery moved the box and wrote the result this run answers |
| 387 | // with — an apply on top of that would overwrite that result and |
| 388 | // restart the edge a second time. The agent re-touches the marker |
| 389 | // if it still wants the update, and by then it can read the result. |
| 390 | return nil |
| 391 | } |
| 392 | req, err := a.readRequest() |
| 393 | if errors.Is(err, os.ErrNotExist) { |
| 394 | return nil |
| 395 | } |
| 396 | if err != nil { |
| 397 | a.ops.logf("system apply: request: %v", err) |
| 398 | return nil // nothing to record against: the request itself is unreadable |
| 399 | } |
| 400 | // A tuple this box has already rolled back is not tried again: the |
| 401 | // build failed its gate here, and re-applying it on every marker touch |
| 402 | // would restart the edge in a loop. Only those two outcomes stick — |
| 403 | // refused, deferred and noted are all worth another go. |
| 404 | if r, _ := readSystemResult(a.optDir); r.matches(a.channel, req.Version, req.PromotedAt) && |
| 405 | (r.Outcome == systemOutcomeRolledBack || r.Outcome == systemOutcomeRollbackFailed) { |
| 406 | a.ops.logf("system apply: %s %s was already %s on this box; not applying it again", a.channel, req.Version, r.Outcome) |
| 407 | return nil |
| 408 | } |
| 409 | p, err := a.verifyPointer(ctx) |
| 410 | if err != nil { |
| 411 | return a.record(req, err) |
| 412 | } |
| 413 | // The moves protocol floor (moves spec §11), written to disk from the |
| 414 | // store before anything is staged, so every check below and every later |
| 415 | // recovery enforces what is published. |
| 416 | if err := a.armFloor(ctx); err != nil { |
| 417 | return a.record(req, err) |
| 418 | } |
| 419 | if req.Version != p.Current.Version || !req.PromotedAt.Equal(p.PromotedAt) { |
| 420 | return a.record(req, fmt.Errorf("%w: request names %s@%s, pointer is %s@%s", errSystemRefused, |
| 421 | req.Version, req.PromotedAt.Format(time.RFC3339), p.Current.Version, p.PromotedAt.Format(time.RFC3339))) |
| 422 | } |
| 423 | seen, err := readSystemSeen(a.optDir) |
| 424 | if err != nil { |
| 425 | return a.record(req, fmt.Errorf("%w: seen.json unreadable: %v", errSystemRefused, err)) |
| 426 | } |
| 427 | if wm, ok := seen[a.channel]; ok && p.PromotedAt.Before(wm) { |
| 428 | return a.record(req, fmt.Errorf("%w: replay — pointer promoted %s is older than the %s watermark %s", errSystemRefused, |
| 429 | p.PromotedAt.Format(time.RFC3339), a.channel, wm.Format(time.RFC3339))) |
| 430 | } |
| 431 | seen[a.channel] = p.PromotedAt |
| 432 | if err := writeJSONDurable(a.file("seen.json"), seen, 0o644); err != nil { |
| 433 | return err |
| 434 | } |
| 435 | if req.NoOp || p.Current.Version == a.onDiskBuild(ctx) { |
| 436 | return a.writeResult(req, systemOutcomeNoted, "") |
| 437 | } |
| 438 | // The claim release floor's transaction (Plan 8) first, as on main; |
| 439 | // then the moves activation check. Either one refusing refuses; a |
| 440 | // moves refusal after beginGuard leaves no journal, so Run's deferred |
| 441 | // checkGuardRecovery closes the guard transaction as it does for any |
| 442 | // refusal before the journal. |
| 443 | if err := a.beginGuard(ctx, p); err != nil { |
| 444 | return err |
| 445 | } |
| 446 | // The install's activation check, before the candidate is copied in or |
| 447 | // run at all: a build below the floor, or a downgrade while a row is |
| 448 | // held, is refused with the box exactly as it was. |
| 449 | if err := a.activationAllowed(ctx, p.Current.Version); err != nil { |
| 450 | return a.record(req, fmt.Errorf("%w: %v", errSystemRefused, err)) |
| 451 | } |
| 452 | staged, err := a.stageCandidate(ctx, p, req) |
| 453 | if err != nil { |
| 454 | return a.record(req, fmt.Errorf("%w: %v", errSystemRefused, err)) |
| 455 | } |
| 456 | return a.install(ctx, req, p, staged) |
| 457 | } |
| 458 | |
| 459 | // waitForLock is --recover's bounded wait for the apply lock: poll |
| 460 | // systemRecoverLockPoll apart for up to systemRecoverLockWait, then give up. |
| 461 | // It reports whether it got the lock; an unsupported platform is the one |
| 462 | // error it returns. |
| 463 | // |
| 464 | // The bound is the fix for a real deadlock. An update run holds this lock |
| 465 | // across `systemctl restart carlos-edge`, and carlos-system-recover.service |
| 466 | // is ordered Before=carlos-edge.service — so if a recover unit's start job |
| 467 | // is queued while an apply is mid-restart, systemd makes the edge's restart |
| 468 | // wait on the recovery, and a recovery that blocked on the lock would wait |
| 469 | // on the apply that is waiting on the restart. Both then sit there until |
| 470 | // their unit timeouts fire. There is nothing for a recovery to do in that |
| 471 | // window anyway: the apply owns the journal it would read, and finishes by |
| 472 | // recovering from it itself if it dies. |
| 473 | func (a *systemApplier) waitForLock(lock *os.File) (bool, error) { |
| 474 | deadline := a.ops.now().Add(systemRecoverLockWait) |
| 475 | for { |
| 476 | err := flockExclusiveNonblock(lock) |
| 477 | if err == nil { |
| 478 | return true, nil |
| 479 | } |
| 480 | if errors.Is(err, errSystemUnsupported) { |
| 481 | return false, err |
| 482 | } |
| 483 | if !a.ops.now().Before(deadline) { |
| 484 | return false, nil |
| 485 | } |
| 486 | a.ops.sleep(systemRecoverLockPoll) |
| 487 | } |
| 488 | } |
| 489 | |
| 490 | // install is steps 6–12: decide .prev, stage both unit sets, then the |
| 491 | // box-changing dance — each step journaled before it happens, and any |
| 492 | // failure from the quiesce on going to the rollback of step 13. |
| 493 | func (a *systemApplier) install(ctx context.Context, req systemRequest, p *release.SystemPointer, staged string) error { |
| 494 | target := p.Current.Version |
| 495 | if err := a.guardImage(staged); err != nil { |
| 496 | return err |
| 497 | } |
| 498 | prevOK, err := a.decidePrev(ctx) |
| 499 | if err != nil { |
| 500 | return a.record(req, fmt.Errorf("%w: %v", errSystemRefused, err)) |
| 501 | } |
| 502 | if !prevOK { |
| 503 | return a.record(req, fmt.Errorf("%w: no known-good carlos.prev to fall back to", errSystemRefused)) |
| 504 | } |
| 505 | newDir, prevDir := filepath.Join(a.stagingDir(), "new"), filepath.Join(a.stagingDir(), "prev") |
| 506 | newPlan, newUnits, err := a.stagePlan(ctx, staged, newDir) |
| 507 | if err != nil { |
| 508 | return a.record(req, fmt.Errorf("%w: candidate bootstrap: %v", errSystemRefused, err)) |
| 509 | } |
| 510 | // A release before the instance layout ships no data-instances.mount, |
| 511 | // so installing it disables the mount units and then serves every |
| 512 | // nested row from an empty mountpoint directory (design §8.2, "That |
| 513 | // order is enforced"). The candidate's own staged unit set is the |
| 514 | // evidence of which kind of release it is. Refused here, before the |
| 515 | // journal, so a refusal leaves the box exactly as it was. |
| 516 | // |
| 517 | // The unit set is not the whole story: v0.20.0 to v0.24.2 ship the |
| 518 | // mounts and still cannot wake a nested row (their edge unit keeps |
| 519 | // RestrictSUIDSGID=, so the agent's openat2 fails; release.LayoutFloor), |
| 520 | // and `carlos system update --file` reaches this guard with no console |
| 521 | // check ahead of it. So a candidate whose reported version parses and |
| 522 | // is below the floor is refused on the same terms. A version that |
| 523 | // cannot be parsed (a local build named by a sha) is judged by its |
| 524 | // unit set alone, as before; docs/site/cli.md says so. |
| 525 | if !slices.Contains(newUnits, layoutMountUnit) || release.BelowFloor(target, release.LayoutFloor) { |
| 526 | if reasons := a.layoutReversalReasons(); len(reasons) > 0 { |
| 527 | return a.record(req, fmt.Errorf("%w: "+systemApplyLayoutRefusal, errSystemRefused, target, strings.Join(reasons, ", "))) |
| 528 | } |
| 529 | } |
| 530 | // Only the unit set is taken from the return: a rollback picks this |
| 531 | // staging up as it stands, plan.json and all, rather than exec'ing the |
| 532 | // previous binary again on a box that is already unhealthy. |
| 533 | _, prevUnits, err := a.stagePlan(ctx, systemPrevPath(a.optDir), prevDir) |
| 534 | if err != nil { |
| 535 | return a.record(req, fmt.Errorf("%w: carlos.prev bootstrap: %v", errSystemRefused, err)) |
| 536 | } |
| 537 | // Region HA (spec 1 §5.10): the marker, then the participation check, |
| 538 | // before the first step that changes the box. Either failing defers the |
| 539 | // update, which the agent retries; the box is left exactly as it was. |
| 540 | if err := a.beginExecutableChange(ctx, target, req.PromotedAt); err != nil { |
| 541 | if !errors.Is(err, errSystemDeferred) { |
| 542 | err = fmt.Errorf("%w: update-begun marker: %v", errSystemDeferred, err) |
| 543 | } |
| 544 | return a.record(req, err) |
| 545 | } |
| 546 | // The release on disk, for the activation guard at the rename: read now, |
| 547 | // because reading it runs the binary and the guard holds locks. |
| 548 | running := a.onDiskBuild(ctx) |
| 549 | // PromotedAt rides along for the whole transaction: it is half of the |
| 550 | // tuple a recovery has to answer for. |
| 551 | j := systemJournal{Phase: systemPhaseInstalling, Target: target, PromotedAt: req.PromotedAt, PrevBuild: a.prevBuild(ctx), |
| 552 | UnitsAdded: unitsMinus(newUnits, prevUnits), UnitsRemoved: unitsMinus(prevUnits, newUnits), StartedAt: a.ops.now().UTC()} |
| 553 | if err := a.journal(j); err != nil { |
| 554 | return err |
| 555 | } |
| 556 | fail := func(step string, err error) error { |
| 557 | return a.rollback(ctx, req, fmt.Sprintf("%s: %v", step, err), &j) |
| 558 | } |
| 559 | // Stop the root passes that exec the old binary against the old edge |
| 560 | // before the rename; one that is already running is killed, and those |
| 561 | // passes are idempotent. Which of their triggers were active is kept, so |
| 562 | // a refused rename can put them back. |
| 563 | prior := a.quiesce() |
| 564 | // The activation rule again, at the step it guards (moves spec §11): the |
| 565 | // check before staging can have gone stale, since the agent may have |
| 566 | // raised the floor or held a row since. activationGuard holds the floor |
| 567 | // and the holds still across the rename. |
| 568 | renamed := false |
| 569 | if err := a.activationGuard(ctx, target, running, func(ctx context.Context) error { |
| 570 | if err := a.rootProtocolAllowsBinary(staged); err != nil { |
| 571 | return err |
| 572 | } |
| 573 | renamed = true |
| 574 | return os.Rename(staged, systemBinaryPath(a.optDir)) |
| 575 | }); err != nil { |
| 576 | if !renamed { |
| 577 | return a.abandonInstall(req, err, prior) |
| 578 | } |
| 579 | return fail("install", err) |
| 580 | } |
| 581 | if err := os.Chown(systemBinaryPath(a.optDir), 0, 0); err != nil && !errors.Is(err, os.ErrPermission) { |
| 582 | a.ops.logf("system apply: chown root: %v", err) |
| 583 | } |
| 584 | j.Phase = systemPhaseInstalled |
| 585 | if err := a.journal(j); err != nil { |
| 586 | return fail("journal", err) |
| 587 | } |
| 588 | if err := a.installUnits(ctx, newDir, j.UnitsRemoved); err != nil { |
| 589 | return fail("units", err) |
| 590 | } |
| 591 | j.Phase = systemPhaseUnits |
| 592 | if err := a.journal(j); err != nil { |
| 593 | return fail("journal", err) |
| 594 | } |
| 595 | // The control service before the edge; see restartInstanceControl for |
| 596 | // why that order, and why a restart that may drop a wake is safe here. |
| 597 | if err := a.restartInstanceControl(); err != nil { |
| 598 | return fail("instance-control restart", err) |
| 599 | } |
| 600 | if err := a.restartEdge(); err != nil { |
| 601 | return fail("restart", err) |
| 602 | } |
| 603 | a.restartRelay() |
| 604 | j.Phase = systemPhaseRestarted |
| 605 | if err := a.journal(j); err != nil { |
| 606 | return fail("journal", err) |
| 607 | } |
| 608 | if err := a.gate(ctx, target); err != nil { |
| 609 | return fail("gate", err) |
| 610 | } |
| 611 | if err := a.acceptRootFloorHealthy(ctx); err != nil { |
| 612 | return fail("gate", err) |
| 613 | } |
| 614 | j.Phase = systemPhaseGated |
| 615 | if err := a.journal(j); err != nil { |
| 616 | return fail("journal", err) |
| 617 | } |
| 618 | // Outside activationGuard, deliberately: the floor protects activation, |
| 619 | // and the guarded rename above was this install's activation. A floor |
| 620 | // raised since applies to the next one. (Recovery's forward finish is |
| 621 | // different: it has no rename of its own, so the finish is its |
| 622 | // activation, and it is guarded.) |
| 623 | return a.finish(ctx, req, target, newPlan) |
| 624 | } |
| 625 | |
| 626 | // abandonInstall undoes an install that the activation guard refused at the |
| 627 | // rename, when nothing but the quiesce has happened: the binary, the units |
| 628 | // and carlos.prev are all as they were. |
| 629 | // |
| 630 | // The paused triggers are started again FIRST, with the journal still |
| 631 | // open, and the journal is removed only once they are (fix round 2). When |
| 632 | // starting them fails, what is owed goes to rearm-pending.json, durably, |
| 633 | // before the journal goes, and the run fails with a reason that says so; |
| 634 | // every later run starts with rearmPending, and this failure fires the |
| 635 | // recover unit at once. The journal itself is not what is kept: while it |
| 636 | // exists the system units pick carlos.prev for that recovery, and a |
| 637 | // carlos.prev older than this build reads an install journal as an |
| 638 | // interrupted install and restores itself over a binary that never changed. |
| 639 | // Only when even the retry state cannot be written does the journal stay. |
| 640 | func (a *systemApplier) abandonInstall(req systemRequest, refusal error, prior []string) error { |
| 641 | // A guard that could not get the floor lock is deferred (retried at the |
| 642 | // next run); every other stop is a refusal. |
| 643 | kind := errSystemRefused |
| 644 | var deferred activationDeferred |
| 645 | if errors.As(refusal, &deferred) { |
| 646 | kind = errSystemDeferred |
| 647 | } |
| 648 | rerr := a.rearm(prior) |
| 649 | if rerr == nil { |
| 650 | a.removeJournal() |
| 651 | return a.record(req, fmt.Errorf("%w: %v", kind, refusal)) |
| 652 | } |
| 653 | reason := fmt.Errorf("%w: %v; the root passes paused for the install could not be started again: %v. the next apply or recovery starts them", |
| 654 | kind, refusal, rerr) |
| 655 | if err := a.notePendingRearm(prior); err != nil { |
| 656 | reason = fmt.Errorf("%w: %v; the root passes paused for the install could not be started again: %v, and the retry could not be recorded: %v. the journal stays for the next recovery", |
| 657 | kind, refusal, rerr, err) |
| 658 | } else { |
| 659 | a.removeJournal() |
| 660 | } |
| 661 | _ = a.record(req, reason) |
| 662 | return fmt.Errorf("system apply: %s: %v", req.Version, reason) |
| 663 | } |
| 664 | |
| 665 | // rearmPendingFile records root-pass triggers a refused step paused and |
| 666 | // could not start again (abandonInstall), for the next run to start. |
| 667 | const rearmPendingFile = "rearm-pending.json" |
| 668 | |
| 669 | type rearmPending struct { |
| 670 | Units []string `json:"units"` |
| 671 | At time.Time `json:"at"` |
| 672 | } |
| 673 | |
| 674 | // notePendingRearm adds units to rearm-pending.json, durably. |
| 675 | func (a *systemApplier) notePendingRearm(units []string) error { |
| 676 | var p rearmPending |
| 677 | if err := readJSONFile(a.file(rearmPendingFile), &p); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 678 | p = rearmPending{} |
| 679 | } |
| 680 | for _, u := range units { |
| 681 | if !slices.Contains(p.Units, u) { |
| 682 | p.Units = append(p.Units, u) |
| 683 | } |
| 684 | } |
| 685 | p.At = a.ops.now().UTC() |
| 686 | return writeJSONDurable(a.file(rearmPendingFile), p, 0o600) |
| 687 | } |
| 688 | |
| 689 | // rearmPending starts what rearm-pending.json records and removes it once |
| 690 | // they are started. A failure is logged and the file stays for the next run. |
| 691 | func (a *systemApplier) rearmPending() { |
| 692 | var p rearmPending |
| 693 | if err := readJSONFile(a.file(rearmPendingFile), &p); err != nil { |
| 694 | if !errors.Is(err, os.ErrNotExist) { |
| 695 | a.ops.logf("system apply: %s: %v", rearmPendingFile, err) |
| 696 | } |
| 697 | return |
| 698 | } |
| 699 | if err := a.rearm(p.Units); err != nil { |
| 700 | a.ops.logf("system apply: the root passes paused by a refused install could not be started again: %v; the next run tries again", err) |
| 701 | return |
| 702 | } |
| 703 | if err := removeDurable(a.file(rearmPendingFile)); err != nil { |
| 704 | a.ops.logf("system apply: remove %s: %v", rearmPendingFile, err) |
| 705 | } |
| 706 | } |
| 707 | |
| 708 | // quiesce stops the root passes (systemQuiesceUnits) and returns the |
| 709 | // timers and paths among them that were active, for rearm to start again |
| 710 | // if the step they were paused for is refused. When their state cannot be |
| 711 | // read, all of them are returned: active is what an armed box has. |
| 712 | func (a *systemApplier) quiesce() []string { |
| 713 | var triggers []string |
| 714 | for _, u := range systemQuiesceUnits { |
| 715 | if strings.HasSuffix(u, ".timer") || strings.HasSuffix(u, ".path") { |
| 716 | triggers = append(triggers, u) |
| 717 | } |
| 718 | } |
| 719 | prior := triggers |
| 720 | // is-active exits non-zero when any unit is not active; the output still |
| 721 | // has one state per unit, in order. |
| 722 | out, _ := a.ops.systemctl(append([]string{"is-active"}, triggers...)...) |
| 723 | if states := strings.Fields(out); len(states) == len(triggers) { |
| 724 | prior = nil |
| 725 | for i, u := range triggers { |
| 726 | if states[i] == "active" || states[i] == "activating" || states[i] == "reloading" { |
| 727 | prior = append(prior, u) |
| 728 | } |
| 729 | } |
| 730 | } |
| 731 | if _, err := a.ops.systemctl(append([]string{"stop"}, systemQuiesceUnits...)...); err != nil { |
| 732 | a.ops.logf("system apply: quiesce: %v (continuing)", err) |
| 733 | } |
| 734 | return prior |
| 735 | } |
| 736 | |
| 737 | // rearm starts units again (nothing to do for none). |
| 738 | func (a *systemApplier) rearm(units []string) error { |
| 739 | if len(units) == 0 { |
| 740 | return nil |
| 741 | } |
| 742 | _, err := a.ops.systemctl(append([]string{"start"}, units...)...) |
| 743 | return err |
| 744 | } |
| 745 | |
| 746 | // finish is steps 11–12, and what recovery runs forward from phase gated. |
| 747 | func (a *systemApplier) finish(ctx context.Context, req systemRequest, target string, plan bootstrapPlan) error { |
| 748 | reason := "" |
| 749 | if err := a.arm(plan); err != nil { |
| 750 | // The gate passed, so this run is an apply and rolling back now |
| 751 | // would undo a build that proved itself. But `enable` is one |
| 752 | // batched call: one bad unit leaves the box on the new binary with |
| 753 | // nothing armed, and a fleet-visible reason is the only way anyone |
| 754 | // finds out. |
| 755 | a.ops.logf("system apply: arm: %v", err) |
| 756 | reason = "armed with errors: " + err.Error() |
| 757 | } |
| 758 | if err := writeJSONDurable(a.file("applied.json"), systemApplied{Version: target, Channel: a.channel, At: a.ops.now().UTC()}, 0o644); err != nil { |
| 759 | return err |
| 760 | } |
| 761 | if err := a.writeResult(req, systemOutcomeApplied, reason); err != nil { |
| 762 | return err |
| 763 | } |
| 764 | return a.closeGuardJournal(ctx) |
| 765 | } |
| 766 | |
| 767 | // removeJournal closes the transaction, durably: an unsynced removal can |
| 768 | // come back after a power loss, and a journal that outlives the run it |
| 769 | // belongs to is one the next run recovers from. |
| 770 | func (a *systemApplier) removeJournal() { |
| 771 | os.Remove(a.file("journal.json")) |
| 772 | if err := syncDir(a.rootDir()); err != nil { |
| 773 | a.ops.logf("system apply: fsync %s: %v", a.rootDir(), err) |
| 774 | } |
| 775 | } |
| 776 | |
| 777 | // decidePrev refreshes carlos.prev from the on-disk binary only when that |
| 778 | // binary is known good — applied.json names it, or nothing has ever been |
| 779 | // applied and there is no .prev (a hand-rolled box whose running binary |
| 780 | // passed the script's own post-condition). Otherwise it leaves .prev alone |
| 781 | // and reports whether there is one to fall back to at all. |
| 782 | // |
| 783 | // The backup is deliberately NOT journaled, though every other box-changing |
| 784 | // step is. copyFileDurable is whole-or-nothing — it writes carlos.prev.tmp |
| 785 | // and renames — so a kill here leaves either the old .prev or the new one, |
| 786 | // never a torn file, and a stray .prev.tmp is truncated by the next write. |
| 787 | // A journal would make it WORSE: a phase written before the copy is a phase |
| 788 | // the exec shim in carlos-system-recover.service reads at the next boot, |
| 789 | // and it would then run a .prev that is one release older than the box, for |
| 790 | // a whole apply, on the strength of a backup that had nothing to recover. |
| 791 | // (recoverIfNeeded still knows systemPhaseBackingUp: journals written by |
| 792 | // the version that did write one outlive the upgrade to this one.) |
| 793 | func (a *systemApplier) decidePrev(ctx context.Context) (bool, error) { |
| 794 | prev := systemPrevPath(a.optDir) |
| 795 | if err := a.guardImage(systemBinaryPath(a.optDir)); err != nil { |
| 796 | return false, err |
| 797 | } |
| 798 | _, prevErr := os.Stat(prev) |
| 799 | applied, err := readSystemApplied(a.optDir) |
| 800 | if err != nil { |
| 801 | return false, err |
| 802 | } |
| 803 | disk := a.onDiskBuild(ctx) |
| 804 | known := (applied != nil && applied.Version == disk && disk != "") || (applied == nil && prevErr != nil) |
| 805 | if !known { |
| 806 | if prevErr == nil { |
| 807 | if err := a.guardImage(prev); err != nil { |
| 808 | return false, err |
| 809 | } |
| 810 | } |
| 811 | return prevErr == nil, a.notePrev(ctx, false, "") |
| 812 | } |
| 813 | if err := a.notePrev(ctx, true, disk); err != nil { |
| 814 | return false, err |
| 815 | } |
| 816 | return true, nil |
| 817 | } |
| 818 | |
| 819 | // notePrev is decidePrev's half of the moves floor (moves spec §11), under |
| 820 | // the lock the adopt pass's judge takes too. With replace, carlos.prev is |
| 821 | // refreshed from the on-disk binary (version is what that binary reported): |
| 822 | // the sidecar and the verdict of the carlos.prev being replaced are removed |
| 823 | // first, so a crash mid-copy leaves a carlos.prev that nothing vouches for |
| 824 | // (refused while a floor is armed), never one vouched for by its |
| 825 | // predecessor's files. Then the sidecar is written and judgePrev runs, all |
| 826 | // before the journal. |
| 827 | // |
| 828 | // The copy's own failure is decidePrev's error as it always was. The floor |
| 829 | // steps are best-effort while no floor is armed (logged, never fatal: an |
| 830 | // unarmed deployment must update exactly as main does), and hard once one |
| 831 | // is (floorStep). |
| 832 | func (a *systemApplier) notePrev(ctx context.Context, replace bool, version string) error { |
| 833 | var copyErr error |
| 834 | ran := false |
| 835 | steps := func() error { |
| 836 | ran = true |
| 837 | if replace { |
| 838 | if err := a.floorStep(errors.Join(removeDurable(prevAllowedPath(a.optDir)), removeDurable(seedsPrevAllowedPath(a.optDir)), removeDurable(prevVersionPath(a.optDir)))); err != nil { |
| 839 | return err |
| 840 | } |
| 841 | if copyErr = copyFileDurable(systemBinaryPath(a.optDir), systemPrevPath(a.optDir), 0o755); copyErr != nil { |
| 842 | return nil |
| 843 | } |
| 844 | if version != "" { |
| 845 | if err := a.floorStep(writePrevVersion(a.optDir, version)); err != nil { |
| 846 | return err |
| 847 | } |
| 848 | } |
| 849 | } |
| 850 | if id, required, err := a.rootMoves.read(); err != nil { |
| 851 | return err |
| 852 | } else if required { |
| 853 | f, err := openRootMovesFloors(a.optDir, id, false) |
| 854 | if err != nil { |
| 855 | return err |
| 856 | } |
| 857 | defer f.close() |
| 858 | j := rootFloorJudge{floors: f, current: systemBinaryPath(a.optDir), prev: systemPrevPath(a.optDir), idle: rootFloorDowngradeIdle, binary: func(path string) (rootFloorBinary, error) { return readRootFloorBinary(a.optDir, path) }} |
| 859 | return errors.Join(j.judge(ctx), judgeRootSeedsPrev(ctx, a.optDir, id)) |
| 860 | } |
| 861 | return a.floorStep(errors.Join(judgePrevWith(a.optDir, a.dataDir, a.freezeHolds, func() string { return a.prevBuild(ctx) }), judgeSeedsPrev(a.optDir, a.dataDir))) |
| 862 | } |
| 863 | err := withPrevLock(a.optDir, steps) |
| 864 | if !ran { |
| 865 | // The lock itself failed. Armed, that is the apply's error; with no |
| 866 | // floor the steps run anyway, because the backup is main's step. |
| 867 | if err = a.floorStep(err); err != nil { |
| 868 | return err |
| 869 | } |
| 870 | err = steps() |
| 871 | } |
| 872 | if copyErr != nil { |
| 873 | return copyErr |
| 874 | } |
| 875 | return err |
| 876 | } |
| 877 | |
| 878 | // floorArmed reports whether a moves protocol floor is armed on this box: |
| 879 | // a local floor file that reads, or one that is there and cannot be read |
| 880 | // (armed, value unknown). An armed seeds floor (seedsfloor.go) counts too, |
| 881 | // since the floor's steps write and judge both. |
| 882 | func (a *systemApplier) floorArmed() bool { |
| 883 | f, err := readEffectiveMovesFloor(a.rootMoves, a.optDir, a.dataDir) |
| 884 | sf, se := readEffectiveSeedsFloor(a.rootMoves, a.optDir, a.dataDir) |
| 885 | return err != nil || f != nil || se != nil || sf != nil |
| 886 | } |
| 887 | |
| 888 | // floorStep is the dormancy rule for the floor's own steps (moves Task 13, |
| 889 | // ruling F15): while no floor is armed a failure is a log line and the |
| 890 | // apply goes on as main's would; once one is, it is the apply's error. |
| 891 | func (a *systemApplier) floorStep(err error) error { |
| 892 | if err == nil { |
| 893 | return nil |
| 894 | } |
| 895 | if a.floorArmed() { |
| 896 | return err |
| 897 | } |
| 898 | a.ops.logf("system apply: %v (no moves protocol floor is armed; continuing)", err) |
| 899 | return nil |
| 900 | } |
| 901 | |
| 902 | // armFloor writes the published floor to this box's local floor files |
| 903 | // (syncLocalFloor), first thing once the store is open. What is published |
| 904 | // and could not be written down defers the apply: the box would otherwise |
| 905 | // enforce less than the floor. Anything else (the store not answering, no |
| 906 | // object, an unusable one) keeps what is on disk and is one log line, so a |
| 907 | // deployment that never published a floor updates exactly as main does. |
| 908 | // The seeds floor is armed first, by the same rule (armSeedsFloor). |
| 909 | func (a *systemApplier) armFloor(ctx context.Context) error { |
| 910 | if id, required, err := a.rootMoves.read(); err != nil { |
| 911 | return err |
| 912 | } else if required { |
| 913 | if err := refreshRootMovesFloor(ctx, a.store, a.optDir, a.dataDir, id); err != nil { |
| 914 | return err |
| 915 | } |
| 916 | return a.armSeedsFloor(ctx) |
| 917 | } |
| 918 | |
| 919 | if err := a.armSeedsFloor(ctx); err != nil { |
| 920 | return err |
| 921 | } |
| 922 | f, err := syncLocalFloor(ctx, a.store, a.dataDir) |
| 923 | if err == nil { |
| 924 | return nil |
| 925 | } |
| 926 | var fw *floorWriteError |
| 927 | if errors.As(err, &fw) { |
| 928 | return fmt.Errorf("%w: %s", errSystemDeferred, fmt.Sprintf("moves protocol floor: could not write it on this box: %v", fw.cause)) |
| 929 | } |
| 930 | onDisk := "none" |
| 931 | if f != nil { |
| 932 | onDisk = f.MinRelease |
| 933 | } |
| 934 | a.ops.logf("system apply: moves protocol floor: %v (the floor on this box stays %s)", err, onDisk) |
| 935 | return nil |
| 936 | } |
| 937 | |
| 938 | // activationAllowed is the applier's own rule for every path that puts an |
| 939 | // executable in place (moves spec §11): the install (for the candidate), |
| 940 | // and restoreFiles (for carlos.prev, by its sidecar), which both rollback |
| 941 | // and recovery go through. Two halves, both only once a floor is armed: |
| 942 | // |
| 943 | // - the floor: version below the armed min_release is refused. The floor |
| 944 | // is read from the local file alone, never the store: boot recovery |
| 945 | // runs before the network. |
| 946 | // - the hold: while any registry row is held by an operation, a version |
| 947 | // below the release running now is refused (§11: a participant in a |
| 948 | // held operation does not downgrade). |
| 949 | // |
| 950 | // With no floor armed it reads neither the registry nor any version and |
| 951 | // allows, with one log line: main's behaviour. |
| 952 | // |
| 953 | // The seeds floor (seedsActivationRefusal) is checked first, on every one |
| 954 | // of these paths and the same way as the floor half; it has no hold half. |
| 955 | // |
| 956 | // This is the EARLY check, made before anything is staged so a refused |
| 957 | // build is never copied in or run. It can go stale by the time the build is |
| 958 | // put in place, so the step that puts it there runs under activationGuard, |
| 959 | // which applies the same rule again with its inputs held still. |
| 960 | // |
| 961 | // How the hold half relates to systemparticipation.go (region HA H14): that |
| 962 | // guard defers any executable change, up or down, while this box takes |
| 963 | // part in a claim operation (.hostexec), and only under CARLOS_REGION_HA, |
| 964 | // and only on the install path. This one refuses only a downgrade, while a |
| 965 | // moves hold_op row exists, once a floor is armed, and on every activating |
| 966 | // path, rollback and recovery included. They overlap on a downgrade |
| 967 | // install during a move that also runs a claim operation; there both run |
| 968 | // (this check before staging, H14's marker just before the journal) and |
| 969 | // either refusal stands. Neither reads or relaxes the other. |
| 970 | func (a *systemApplier) activationAllowed(ctx context.Context, version string) error { |
| 971 | return a.activationRule(version, func() string { return a.onDiskBuild(ctx) }) |
| 972 | } |
| 973 | |
| 974 | // activationRefused is a refusal by the activation rule, told apart from a |
| 975 | // failure of the step it guards. |
| 976 | type activationRefused struct{ error } |
| 977 | |
| 978 | func (e activationRefused) Unwrap() error { return e.error } |
| 979 | |
| 980 | func refuse(format string, args ...any) error { |
| 981 | return activationRefused{fmt.Errorf(format, args...)} |
| 982 | } |
| 983 | |
| 984 | // activationDeferred is the guard unable to get floor.lock because another |
| 985 | // process holds it: a writer is active and may be about to publish a floor. |
| 986 | // The step is put off to the next run, never taken unlocked (fix round 3). |
| 987 | type activationDeferred struct{ error } |
| 988 | |
| 989 | func (e activationDeferred) Unwrap() error { return e.error } |
| 990 | |
| 991 | // activationGuard is the ONE guard for every step that makes an executable |
| 992 | // this box's (moves spec §11; fix round 3): the install's rename, the |
| 993 | // restore's rename, and each finish of a journal that has no rename of its |
| 994 | // own (recovery forward and backward, and rollback's). Nothing else renames |
| 995 | // an executable into place or finishes a journal; the scan in |
| 996 | // activationguard_test.go fails if something does. |
| 997 | // |
| 998 | // After release and participation exclusion, it takes floor.lock, creating .moves on a box with |
| 999 | // no floor yet, so no writer can arm or raise a floor until fn has run. |
| 1000 | // Then, with a floor armed, it always freezes the held rows |
| 1001 | // (registry.WithHoldsFrozen), whether or not target is the running release, |
| 1002 | // and checks the floor and the hold rule inside, immediately before fn. |
| 1003 | // With no floor armed, under the lock none can appear, the hold rule does |
| 1004 | // not apply, and the registry is not consulted (index D13: an unarmed |
| 1005 | // deployment behaves as main). |
| 1006 | // |
| 1007 | // Lock failures: |
| 1008 | // - contention (the wait ran out): activationDeferred. Never unlocked: a |
| 1009 | // writer holding the lock may not have published yet. |
| 1010 | // - the lock cannot exist (.moves or floor.lock cannot be created: |
| 1011 | // read-only, full or unwritable data dir) and no floor is armed: one |
| 1012 | // log line and fn runs as main's step would (F15). No writer can arm a |
| 1013 | // floor on such a box either. |
| 1014 | // - anything else, or a floor armed: a refusal. |
| 1015 | // |
| 1016 | // Run holds the apply flock. Census activation takes release exclusion, |
| 1017 | // then participation, then the effective floor and registry write locks. |
| 1018 | // Journal close reuses the held release and participation contexts. |
| 1019 | // Nothing here may take the apply flock. |
| 1020 | // |
| 1021 | // running is the release on disk, read by the caller before any lock, |
| 1022 | // since reading it runs a binary. A refusal or deferral comes back typed; |
| 1023 | // fn's own error comes back as it is. |
| 1024 | func (a *systemApplier) activationGuard(ctx context.Context, target, running string, fn func(context.Context) error) error { |
| 1025 | guard := func(ctx context.Context) error { |
| 1026 | if _, required, err := a.rootMoves.read(); err != nil { |
| 1027 | return err |
| 1028 | } else if required { |
| 1029 | bound, e := readEffectiveMovesFloor(a.rootMoves, a.optDir, a.dataDir) |
| 1030 | if e != nil { |
| 1031 | return e |
| 1032 | } |
| 1033 | if at, _ := moves.ReleaseAtOrAbove(target, running); bound != nil && !at { |
| 1034 | if err := rootFloorDowngradeIdle(ctx); err != nil { |
| 1035 | return err |
| 1036 | } |
| 1037 | } |
| 1038 | } |
| 1039 | step := func() error { |
| 1040 | if a.beforeSwap != nil { |
| 1041 | a.beforeSwap() |
| 1042 | } |
| 1043 | return fn(ctx) |
| 1044 | } |
| 1045 | locked := false |
| 1046 | err := withEffectiveMovesFloorLock(a.rootMoves, a.optDir, a.dataDir, true, func() error { |
| 1047 | locked = true |
| 1048 | if err := a.seedsActivationRefusal(target); err != nil { |
| 1049 | return err |
| 1050 | } |
| 1051 | floor, err := readEffectiveMovesFloor(a.rootMoves, a.optDir, a.dataDir) |
| 1052 | if err != nil { |
| 1053 | return refuse("moves protocol floor: the floor on this box cannot be read: %v", err) |
| 1054 | } |
| 1055 | if floor == nil { |
| 1056 | return step() |
| 1057 | } |
| 1058 | if err := floorRefusal(target, a.activationFloor(floor, false)); err != nil { |
| 1059 | return err |
| 1060 | } |
| 1061 | frozen := false |
| 1062 | err = a.freezeHolds(func(held []registry.Route) error { |
| 1063 | frozen = true |
| 1064 | if err := holdRefusal(target, running, held); err != nil { |
| 1065 | return err |
| 1066 | } |
| 1067 | return step() |
| 1068 | }) |
| 1069 | if !frozen { |
| 1070 | return refuse("a participant in a held operation does not downgrade, and the registry cannot say whether this box is one: %v", err) |
| 1071 | } |
| 1072 | return err |
| 1073 | }) |
| 1074 | if locked { |
| 1075 | return err |
| 1076 | } |
| 1077 | if errors.Is(err, errFloorLockBusy) { |
| 1078 | return activationDeferred{errors.New("moves protocol floor: another writer holds the floor lock on this box; this waits for the next run")} |
| 1079 | } |
| 1080 | if floorLockCannotExist(err) && !a.floorArmed() { |
| 1081 | a.ops.logf("system apply: moves protocol floor lock: %v (no floor is armed, and none can be on this box; continuing)", err) |
| 1082 | return step() |
| 1083 | } |
| 1084 | return refuse("moves protocol floor: the floor on this box cannot be read: %v", err) |
| 1085 | } |
| 1086 | return a.participationDisk().WithParticipation(ctx, func(ctx context.Context, l rootparticipation.Ledger) error { return l.WithTransaction(ctx, guard) }, func() error { return guard(ctx) }) |
| 1087 | } |
| 1088 | |
| 1089 | // floorRefusal is the floor half of the rule. |
| 1090 | func floorRefusal(version string, floor *moves.Floor) error { |
| 1091 | if floor == nil { |
| 1092 | return nil |
| 1093 | } |
| 1094 | if version == "" { |
| 1095 | return refuse("moves protocol floor: carlos.prev records no version, so it is below %s", floor.MinRelease) |
| 1096 | } |
| 1097 | if ok, _ := moves.ReleaseAtOrAbove(version, floor.MinRelease); !ok { |
| 1098 | return refuse("moves protocol floor: %s is below %s", version, floor.MinRelease) |
| 1099 | } |
| 1100 | return nil |
| 1101 | } |
| 1102 | |
| 1103 | // holdRefusal is the hold half: while any row is held, nothing below the |
| 1104 | // running release. The same release, or a later one, is not a downgrade. |
| 1105 | func holdRefusal(version, running string, held []registry.Route) error { |
| 1106 | if len(held) == 0 || version == running { |
| 1107 | return nil |
| 1108 | } |
| 1109 | if ok, _ := moves.ReleaseAtOrAbove(version, running); ok { |
| 1110 | return nil |
| 1111 | } |
| 1112 | if running == "" { |
| 1113 | return refuse("a participant in a held operation does not downgrade, and the running release is unknown") |
| 1114 | } |
| 1115 | return refuse("a participant in a held operation does not downgrade: %s is below the running %s", version, running) |
| 1116 | } |
| 1117 | |
| 1118 | // activationRule is the early, unlocked check (activationAllowed, and |
| 1119 | // restoreFiles before its quiesce): the same two halves, read once, so a |
| 1120 | // refused build is never staged and a refused restore never quiesces. It |
| 1121 | // guards nothing by itself; the step it precedes runs under |
| 1122 | // activationGuard. |
| 1123 | func (a *systemApplier) activationRule(version string, running func() string) error { |
| 1124 | if err := a.seedsActivationRefusal(version); err != nil { |
| 1125 | return err |
| 1126 | } |
| 1127 | floor, err := readEffectiveMovesFloor(a.rootMoves, a.optDir, a.dataDir) |
| 1128 | if err != nil { |
| 1129 | return refuse("moves protocol floor: the floor on this box cannot be read: %v", err) |
| 1130 | } |
| 1131 | if floor == nil { |
| 1132 | a.ops.logf("system apply: no moves protocol floor is armed on this box; %q is not checked against one", version) |
| 1133 | return nil |
| 1134 | } |
| 1135 | if err := floorRefusal(version, a.activationFloor(floor, false)); err != nil { |
| 1136 | return err |
| 1137 | } |
| 1138 | run := running() |
| 1139 | if version == run { |
| 1140 | return nil |
| 1141 | } |
| 1142 | if ok, _ := moves.ReleaseAtOrAbove(version, run); ok { |
| 1143 | return nil |
| 1144 | } |
| 1145 | var held []registry.Route |
| 1146 | called := false |
| 1147 | err = a.freezeHolds(func(h []registry.Route) error { called = true; held = h; return nil }) |
| 1148 | if !called { |
| 1149 | return refuse("a participant in a held operation does not downgrade, and the registry cannot say whether this box is one: %v", err) |
| 1150 | } |
| 1151 | return holdRefusal(version, run, held) |
| 1152 | } |
| 1153 | |
| 1154 | // floorAllowsForward is the floor half alone, for recovery's forward path: |
| 1155 | // the gated build is already the one on disk, so only the floor can refuse |
| 1156 | // it. Unarmed: allowed. |
| 1157 | func (a *systemApplier) floorAllowsForward(version string) error { |
| 1158 | if err := a.seedsActivationRefusal(version); err != nil { |
| 1159 | return err |
| 1160 | } |
| 1161 | floor, err := readEffectiveMovesFloor(a.rootMoves, a.optDir, a.dataDir) |
| 1162 | if err != nil { |
| 1163 | return fmt.Errorf("moves protocol floor: the floor on this box cannot be read: %v", err) |
| 1164 | } |
| 1165 | floor = a.activationFloor(floor, false) |
| 1166 | if floor == nil { |
| 1167 | return nil |
| 1168 | } |
| 1169 | if ok, _ := moves.ReleaseAtOrAbove(version, floor.MinRelease); !ok { |
| 1170 | return fmt.Errorf("moves protocol floor: %s is below %s", version, floor.MinRelease) |
| 1171 | } |
| 1172 | return nil |
| 1173 | } |
| 1174 | |
| 1175 | // freezeHolds opens the registry (openRegistry, as the layout guard does) |
| 1176 | // and runs fn with the held rows frozen (registry.WithHoldsFrozen). |
| 1177 | func (a *systemApplier) freezeHolds(fn func(held []registry.Route) error) error { |
| 1178 | open := a.openRegistry |
| 1179 | if open == nil { |
| 1180 | open = func() (*registry.Registry, error) { return openSystemRegistry(a.dataDir) } |
| 1181 | } |
| 1182 | reg, err := open() |
| 1183 | if err != nil { |
| 1184 | return err |
| 1185 | } |
| 1186 | defer reg.Close() |
| 1187 | return reg.WithHoldsFrozen(fn) |
| 1188 | } |
| 1189 | |
| 1190 | func (a *systemApplier) prevBuild(ctx context.Context) string { |
| 1191 | out, err := a.guardRun(ctx, systemVersionWait, systemPrevPath(a.optDir), "version") |
| 1192 | if err != nil { |
| 1193 | return "" |
| 1194 | } |
| 1195 | return carlosVersionOf(out) |
| 1196 | } |
| 1197 | |
| 1198 | // stagePlan runs `<binary> bootstrap -root <dir> -plan-json` and returns |
| 1199 | // that binary's arming plan and the unit files it staged. The plan is left |
| 1200 | // in <dir>/plan.json next to them, so a later step can pick up the whole |
| 1201 | // staging (stagedPlan) without exec'ing the binary a second time. |
| 1202 | func (a *systemApplier) stagePlan(ctx context.Context, binary, dir string) (bootstrapPlan, []string, error) { |
| 1203 | os.RemoveAll(dir) |
| 1204 | if err := os.MkdirAll(dir, 0o700); err != nil { |
| 1205 | return bootstrapPlan{}, nil, err |
| 1206 | } |
| 1207 | out, err := a.guardRun(ctx, systemBootstrapWait, binary, "bootstrap", "-root", dir, "-plan-json") |
| 1208 | if err != nil { |
| 1209 | return bootstrapPlan{}, nil, err |
| 1210 | } |
| 1211 | var plan bootstrapPlan |
| 1212 | if err := json.Unmarshal([]byte(out), &plan); err != nil { |
| 1213 | return bootstrapPlan{}, nil, fmt.Errorf("plan: %w", err) |
| 1214 | } |
| 1215 | entries, err := os.ReadDir(filepath.Join(dir, "etc", "systemd", "system")) |
| 1216 | if err != nil { |
| 1217 | return bootstrapPlan{}, nil, err |
| 1218 | } |
| 1219 | var units []string |
| 1220 | for _, e := range entries { |
| 1221 | if !e.IsDir() { |
| 1222 | units = append(units, e.Name()) |
| 1223 | } |
| 1224 | } |
| 1225 | if err := writeJSONDurable(filepath.Join(dir, "plan.json"), plan, 0o600); err != nil { |
| 1226 | return bootstrapPlan{}, nil, err |
| 1227 | } |
| 1228 | return plan, units, nil |
| 1229 | } |
| 1230 | |
| 1231 | // layoutMountUnit is the unit whose presence in a candidate's staged set |
| 1232 | // marks a release that understands the nested layout. |
| 1233 | const layoutMountUnit = "data-instances.mount" |
| 1234 | |
| 1235 | // systemApplyLayoutRefusal takes the release and the rows in its way. |
| 1236 | const systemApplyLayoutRefusal = "%s predates the instance layout, and this box still has: %s. revert the layout and wait for every row to report flat before installing it" |
| 1237 | |
| 1238 | // layoutReversalReasons opens the registry and lists what stands in the |
| 1239 | // way of a release before the layout. A registry that will not open cannot |
| 1240 | // vouch that nothing is nested, so its error is itself a reason. |
| 1241 | func (a *systemApplier) layoutReversalReasons() []string { |
| 1242 | if _, required, err := a.rootMoves.read(); err != nil { |
| 1243 | return []string{err.Error()} |
| 1244 | } else if required { |
| 1245 | if err := rootFloorDowngradeIdle(context.Background()); err != nil { |
| 1246 | return []string{err.Error()} |
| 1247 | } |
| 1248 | snapshot, err := productionGenerationInventory(context.Background()) |
| 1249 | if err != nil { |
| 1250 | return []string{err.Error()} |
| 1251 | } |
| 1252 | for _, view := range snapshot.Installations { |
| 1253 | if !view.Installation.Empty { |
| 1254 | return []string{errRootFloorHistory.Error()} |
| 1255 | } |
| 1256 | } |
| 1257 | } |
| 1258 | |
| 1259 | open := a.openRegistry |
| 1260 | if open == nil { |
| 1261 | open = func() (*registry.Registry, error) { return openSystemRegistry(a.dataDir) } |
| 1262 | } |
| 1263 | reg, err := open() |
| 1264 | if err != nil { |
| 1265 | return []string{err.Error()} |
| 1266 | } |
| 1267 | defer reg.Close() |
| 1268 | return layoutReversalIncomplete(reg, a.dataDir) |
| 1269 | } |
| 1270 | |
| 1271 | // layoutReversalIncomplete names, once each and sorted, every host a |
| 1272 | // release before the layout would mishandle: a row whose paths are nested |
| 1273 | // by shape (provision.NestedPaths, whatever its flags, so a row that left |
| 1274 | // management is still seen), a row the layout pass's own classification |
| 1275 | // calls fenced or in conflict, and a host with a migration journal whether |
| 1276 | // or not its row still exists. A registry that cannot be listed, or a |
| 1277 | // journal directory that cannot be read, is named as such rather than |
| 1278 | // taken for an empty box. |
| 1279 | func layoutReversalIncomplete(reg *registry.Registry, dataDir string) []string { |
| 1280 | named := map[string]bool{} |
| 1281 | var extra []string |
| 1282 | routes, err := reg.List() |
| 1283 | if err != nil { |
| 1284 | extra = append(extra, err.Error()) |
| 1285 | } |
| 1286 | pass := layout.Pass{Registry: reg, DataDir: dataDir} |
| 1287 | for _, rt := range routes { |
| 1288 | if provision.NestedPaths(dataDir, rt) { |
| 1289 | named[rt.Host] = true |
| 1290 | continue |
| 1291 | } |
| 1292 | if l, _ := pass.Classify(rt, routes); l == activator.LayoutFenced || l == activator.LayoutConflict { |
| 1293 | named[rt.Host] = true |
| 1294 | } |
| 1295 | } |
| 1296 | entries, err := os.ReadDir(filepath.Join(dataDir, "layout")) |
| 1297 | if err != nil && !errors.Is(err, os.ErrNotExist) { |
| 1298 | extra = append(extra, err.Error()) |
| 1299 | } |
| 1300 | for _, e := range entries { |
| 1301 | if host, ok := strings.CutSuffix(e.Name(), ".json"); ok && !strings.HasPrefix(host, ".") && layout.JournalPresent(dataDir, host) { |
| 1302 | named[host] = true |
| 1303 | } |
| 1304 | } |
| 1305 | reasons := make([]string, 0, len(named)+len(extra)) |
| 1306 | for h := range named { |
| 1307 | reasons = append(reasons, h) |
| 1308 | } |
| 1309 | slices.Sort(reasons) |
| 1310 | return append(reasons, extra...) |
| 1311 | } |
| 1312 | |
| 1313 | func (a *systemApplier) journal(j systemJournal) error { |
| 1314 | return writeJSONDurable(a.file("journal.json"), j, 0o600) |
| 1315 | } |
| 1316 | |
| 1317 | // installUnits removes what the target build dropped, copies its units in, |
| 1318 | // then sysusers BEFORE tmpfiles — tmpfiles silently drops a line whose |
| 1319 | // group does not resolve, and carlos-edge never starts again after the |
| 1320 | // next boot — then daemon-reload. Never the staging root's data/: it holds |
| 1321 | // a 0-byte registry.db. |
| 1322 | func (a *systemApplier) installUnits(ctx context.Context, stagedRoot string, remove []string) error { |
| 1323 | claimSettings, settingsErr := readClaimUnitSettings(filepath.Join(a.etcDir, "carlos", "host.env")) |
| 1324 | if settingsErr != nil { |
| 1325 | a.ops.logf("system apply: instance-control unit settings: %v", settingsErr) |
| 1326 | } |
| 1327 | unitDir := filepath.Join(a.etcDir, "systemd", "system") |
| 1328 | if err := os.MkdirAll(unitDir, 0o755); err != nil { |
| 1329 | return err |
| 1330 | } |
| 1331 | for _, u := range remove { |
| 1332 | if _, err := a.ops.systemctl("disable", "--now", u); err != nil { |
| 1333 | a.ops.logf("system apply: disable %s: %v", u, err) |
| 1334 | } |
| 1335 | os.Remove(filepath.Join(unitDir, u)) |
| 1336 | } |
| 1337 | beforeControl, err := os.ReadFile(filepath.Join(unitDir, claimControlUnit)) |
| 1338 | if err != nil && !errors.Is(err, os.ErrNotExist) { |
| 1339 | return err |
| 1340 | } |
| 1341 | // A host.env this build cannot parse says nothing about the claim |
| 1342 | // (6b-N2). A control unit already rendered for an enabled box stays as |
| 1343 | // it is, unrestarted, as adopt leaves it; a flag-off one is rendered as |
| 1344 | // main renders it. |
| 1345 | keepControl := settingsErr != nil && bytes.Contains(beforeControl, []byte(claimControlEnvironment)) |
| 1346 | entries, err := os.ReadDir(filepath.Join(stagedRoot, "etc", "systemd", "system")) |
| 1347 | if err != nil { |
| 1348 | return err |
| 1349 | } |
| 1350 | for _, e := range entries { |
| 1351 | if e.IsDir() { |
| 1352 | continue |
| 1353 | } |
| 1354 | if e.Name() == claimControlUnit { |
| 1355 | if keepControl { |
| 1356 | a.ops.logf("system apply: %s kept as installed: host.env cannot be parsed", claimControlUnit) |
| 1357 | continue |
| 1358 | } |
| 1359 | b, err := os.ReadFile(filepath.Join(stagedRoot, "etc", "systemd", "system", e.Name())) |
| 1360 | if err != nil { |
| 1361 | return err |
| 1362 | } |
| 1363 | if err := writeFileDurable(filepath.Join(unitDir, e.Name()), claimUnitContent(e.Name(), b, claimSettings), 0644); err != nil { |
| 1364 | return err |
| 1365 | } |
| 1366 | continue |
| 1367 | } |
| 1368 | if err := copyFileDurable(filepath.Join(stagedRoot, "etc", "systemd", "system", e.Name()), filepath.Join(unitDir, e.Name()), 0o644); err != nil { |
| 1369 | return err |
| 1370 | } |
| 1371 | } |
| 1372 | sys := filepath.Join(stagedRoot, "etc", "sysusers.d", "carlos.conf") |
| 1373 | if _, err := os.Stat(sys); err == nil { |
| 1374 | if err := os.MkdirAll(filepath.Join(a.etcDir, "sysusers.d"), 0o755); err != nil { |
| 1375 | return err |
| 1376 | } |
| 1377 | dst := filepath.Join(a.etcDir, "sysusers.d", "carlos.conf") |
| 1378 | if err := copyFileDurable(sys, dst, 0o644); err != nil { |
| 1379 | return err |
| 1380 | } |
| 1381 | if _, err := a.ops.run(ctx, systemVersionWait, "systemd-sysusers", dst); err != nil { |
| 1382 | return fmt.Errorf("systemd-sysusers: %w", err) |
| 1383 | } |
| 1384 | } |
| 1385 | // The journal cap (discussion #111). This is how a box that was |
| 1386 | // bootstrapped before the cap existed actually gets it: the drop-in is a |
| 1387 | // named file in the staged etc/ tree, exactly like sysusers and tmpfiles |
| 1388 | // below, and copying it here means the fleet converges on the cap through |
| 1389 | // the self-update relay rather than one box at a time by hand. |
| 1390 | // |
| 1391 | // Stat-guarded like sysusers: a ROLLBACK stages carlos.prev, and a |
| 1392 | // previous build whose bootstrap wrote no drop-in must leave whatever is |
| 1393 | // on the box alone rather than fail the restore. |
| 1394 | // |
| 1395 | // journald is restarted, not just reloaded, because that is the only way |
| 1396 | // it re-reads its configuration; failure is logged and the apply |
| 1397 | // continues, since an uncapped journal must not roll back a good build. |
| 1398 | jrnl := filepath.Join(stagedRoot, "etc", "systemd", "journald.conf.d", journaldConfName) |
| 1399 | if _, err := os.Stat(jrnl); err == nil { |
| 1400 | if err := os.MkdirAll(filepath.Join(a.etcDir, "systemd", "journald.conf.d"), 0o755); err != nil { |
| 1401 | return err |
| 1402 | } |
| 1403 | if err := copyFileDurable(jrnl, filepath.Join(a.etcDir, "systemd", "journald.conf.d", journaldConfName), 0o644); err != nil { |
| 1404 | return err |
| 1405 | } |
| 1406 | if _, err := a.ops.systemctl(journaldReload...); err != nil { |
| 1407 | a.ops.logf("system apply: restart systemd-journald: %v (continuing; the journal cap takes effect at the next restart)", err) |
| 1408 | } |
| 1409 | } |
| 1410 | tmp := filepath.Join(stagedRoot, "etc", "tmpfiles.d", "carlos.conf") |
| 1411 | if err := os.MkdirAll(filepath.Join(a.etcDir, "tmpfiles.d"), 0o755); err != nil { |
| 1412 | return err |
| 1413 | } |
| 1414 | dst := filepath.Join(a.etcDir, "tmpfiles.d", "carlos.conf") |
| 1415 | if err := copyFileDurable(tmp, dst, 0o644); err != nil { |
| 1416 | return err |
| 1417 | } |
| 1418 | if _, err := a.ops.run(ctx, systemVersionWait, "systemd-tmpfiles", "--create", dst); err != nil { |
| 1419 | a.ops.logf("system apply: systemd-tmpfiles: %v (continuing, as the script does)", err) |
| 1420 | } |
| 1421 | if _, err = a.ops.systemctl("daemon-reload"); err != nil { |
| 1422 | return err |
| 1423 | } |
| 1424 | afterControl, err := os.ReadFile(filepath.Join(unitDir, claimControlUnit)) |
| 1425 | if err != nil && !errors.Is(err, os.ErrNotExist) { |
| 1426 | return err |
| 1427 | } |
| 1428 | return restartClaimControl(filepath.Join(unitDir, claimControlUnit), beforeControl, afterControl, a.ops.systemctl) |
| 1429 | } |
| 1430 | |
| 1431 | // restartEdge resets the failure counter FIRST — a unit that hit its start |
| 1432 | // limit refuses restart, and so would the rollback — then restarts. |
| 1433 | func (a *systemApplier) restartEdge() error { |
| 1434 | a.ops.systemctl("reset-failed", systemEdgeUnit) |
| 1435 | _, err := a.ops.systemctl("restart", systemEdgeUnit) |
| 1436 | return err |
| 1437 | } |
| 1438 | |
| 1439 | // restartInstanceControl restarts carlos-instance-control.service, the |
| 1440 | // root service that starts every nested instance (design §5). It is |
| 1441 | // socket-activated and then stays up: on the flagship it had run the |
| 1442 | // 2026-09-16 binary for three weeks when v0.25.1 was installed around it |
| 1443 | // (discussion #256). A control service of the previous release serves the |
| 1444 | // new release's unit templates, which bind sources only the new code |
| 1445 | // writes (/run/carlos-control/exe/<iid>/), so every nested start failed |
| 1446 | // at namespace setup and the probe unit, run by the same stale service, |
| 1447 | // failed the same way: the box reported not ready with the right binary |
| 1448 | // on disk and nothing to say why. |
| 1449 | // |
| 1450 | // Order: after the new units are installed and daemon-reload has run, and |
| 1451 | // BEFORE the edge restarts. The new service then runs its start-up probe |
| 1452 | // against the old edge, which is up, so the verdict it records is real |
| 1453 | // (a probe with no edge cgroup fails on cgroup-ro); and by the time the |
| 1454 | // new agent starts, the service it calls is already the new build, so |
| 1455 | // there is no window in which a new agent asks an old service to start |
| 1456 | // units it cannot. A wake in flight at this moment is lost, and that is |
| 1457 | // safe: the edge restart one step later stops every instance anyway |
| 1458 | // (BindsTo=carlos-edge.service), so the wake fails either way; the |
| 1459 | // systemctl start the dropped request was running is a job systemd |
| 1460 | // finishes on its own; the agent reads the dropped connection as a failed |
| 1461 | // wake and sends the stop, which the socket unit, up throughout, queues |
| 1462 | // for the restarted service. Flat hosts never pass through this service. |
| 1463 | // |
| 1464 | // A build that ships no control service (a downgrade below v0.20, whose |
| 1465 | // unit installUnits has just disabled and removed) has nothing to |
| 1466 | // restart; that is the one case this returns nil without calling |
| 1467 | // systemctl. reset-failed first, as restartEdge does, so a unit that hit |
| 1468 | // its start limit can be restarted at all. |
| 1469 | func (a *systemApplier) restartInstanceControl() error { |
| 1470 | if _, err := os.Stat(filepath.Join(a.etcDir, "systemd", "system", systemControlUnit)); err != nil { |
| 1471 | if errors.Is(err, os.ErrNotExist) { |
| 1472 | return nil |
| 1473 | } |
| 1474 | return err |
| 1475 | } |
| 1476 | a.ops.systemctl("reset-failed", systemControlUnit) |
| 1477 | _, err := a.ops.systemctl("restart", systemControlUnit) |
| 1478 | return err |
| 1479 | } |
| 1480 | |
| 1481 | // restartRelay mirrors restartEdge for the relay (relay spec §3), with one |
| 1482 | // difference that decides its shape: the relay is optional. A box with no |
| 1483 | // /etc/carlos/relay-masters must not be handed a relay it never asked for. |
| 1484 | // |
| 1485 | // What keeps that promise is the unit's own ConditionPathExists, not the |
| 1486 | // verb: on an unarmed box `restart` starts nothing, because the condition |
| 1487 | // fails and systemd records the unit as skipped — which is exactly what |
| 1488 | // bootstrap already relies on when it starts the unit outright. So the verb |
| 1489 | // here is `restart`, the same one restartEdge uses, and `try-restart` is |
| 1490 | // wrong for a reason that only shows up on a bad day: reset-failed turns a |
| 1491 | // crash-looped relay from failed into inactive, and try-restart does nothing |
| 1492 | // at all to an inactive unit. A relay that died at boot would then stay down |
| 1493 | // until the NEXT platform update happened to run arm(), with the box's |
| 1494 | // heartbeat the only sign. |
| 1495 | // |
| 1496 | // The error is still logged and swallowed. A relay that will not start is not |
| 1497 | // a failure of the update — the edge, the binary and the gate are all fine — |
| 1498 | // and `carlos relay status` is where a dead region is meant to be read. |
| 1499 | func (a *systemApplier) restartRelay() { |
| 1500 | a.ops.systemctl("reset-failed", systemRelayUnit) |
| 1501 | if _, err := a.ops.systemctl("restart", systemRelayUnit); err != nil { |
| 1502 | a.ops.logf("system apply: relay restart: %v", err) |
| 1503 | } |
| 1504 | } |
| 1505 | |
| 1506 | // gate: within gateWait the edge is active, answers on loopback, and the |
| 1507 | // installed binary reports the target; then a soak across which NRestarts |
| 1508 | // stays 0 — step 9's reset-failed zeroed it, so anything above 0 is a |
| 1509 | // crash the new build caused — and ActiveEnterTimestampMonotonic does not |
| 1510 | // move. A build that starts, answers, and dies a second later is caught |
| 1511 | // here, and so is one that crashed once and then settled. |
| 1512 | func (a *systemApplier) gate(ctx context.Context, target string) error { |
| 1513 | wait, soak := a.gateWait, a.soak |
| 1514 | if wait <= 0 { |
| 1515 | wait = systemGateWaitFor(a.claims) |
| 1516 | } |
| 1517 | if soak <= 0 { |
| 1518 | soak = systemGateSoak |
| 1519 | } |
| 1520 | deadline := a.ops.now().Add(wait) |
| 1521 | var lastErr error |
| 1522 | for { |
| 1523 | var state string |
| 1524 | state, lastErr = a.gateCheck(ctx, target) |
| 1525 | if lastErr == nil { |
| 1526 | break |
| 1527 | } |
| 1528 | if a.claims { |
| 1529 | if err := a.gateFailFast(state, lastErr); err != nil { |
| 1530 | return err |
| 1531 | } |
| 1532 | } |
| 1533 | if !a.ops.now().Before(deadline) { |
| 1534 | return lastErr |
| 1535 | } |
| 1536 | a.ops.sleep(time.Second) |
| 1537 | } |
| 1538 | before, err := a.edgeRunState() |
| 1539 | if err != nil { |
| 1540 | return err |
| 1541 | } |
| 1542 | if err := before.steady(); err != nil { |
| 1543 | return err |
| 1544 | } |
| 1545 | a.ops.sleep(soak) |
| 1546 | after, err := a.edgeRunState() |
| 1547 | if err != nil { |
| 1548 | return err |
| 1549 | } |
| 1550 | if err := after.steady(); err != nil { |
| 1551 | return err |
| 1552 | } |
| 1553 | if before != after { |
| 1554 | return fmt.Errorf("edge restarted during the soak (%s → %s)", before, after) |
| 1555 | } |
| 1556 | return a.gateOnce(ctx, target) |
| 1557 | } |
| 1558 | |
| 1559 | // systemApplyGate is how RunSystemApply reads claims mode for the gate: the |
| 1560 | // unit's CARLOS_HOST_CLAIM through hostwrite.Enabled, the test the agent |
| 1561 | // and the claim wiring make, and the window that follows from it. |
| 1562 | func systemApplyGate(getenv func(string) string) (bool, time.Duration) { |
| 1563 | claims := hostwrite.Enabled(getenv) |
| 1564 | return claims, systemGateWaitFor(claims) |
| 1565 | } |
| 1566 | |
| 1567 | // systemGateWaitFor is the gate's window: systemClaimsGateWait with |
| 1568 | // CARLOS_HOST_CLAIM on, systemGateWait without. The install's gate and the |
| 1569 | // rollback's gate both use it, so a claims-on box can roll back to a |
| 1570 | // previous build that also needs more than 20 s to listen. |
| 1571 | func systemGateWaitFor(claims bool) time.Duration { |
| 1572 | if claims { |
| 1573 | return systemClaimsGateWait |
| 1574 | } |
| 1575 | return systemGateWait |
| 1576 | } |
| 1577 | |
| 1578 | // gateFailFast is what makes the claims window "wait while the same process |
| 1579 | // is still starting" rather than "wait 120 s for anything". After a failed |
| 1580 | // check it returns an error, ending the gate before its deadline, when the |
| 1581 | // edge unit reports a failed or inactive state (from the is-active the |
| 1582 | // check just made) or NRestarts above 0 (from the same systemctl show the |
| 1583 | // soak reads). A build that crashes rolls back in seconds; one that is |
| 1584 | // still starting, active with no restart, gets the whole window. A show |
| 1585 | // that cannot be read says nothing either way, so the wait goes on to the |
| 1586 | // deadline. Claims-off boxes never call it. |
| 1587 | func (a *systemApplier) gateFailFast(state string, cause error) error { |
| 1588 | switch state { |
| 1589 | case "failed", "inactive": |
| 1590 | return fmt.Errorf("%w (failing fast: %s is %s)", cause, systemEdgeUnit, state) |
| 1591 | } |
| 1592 | rs, err := a.edgeRunState() |
| 1593 | if err != nil { |
| 1594 | return nil |
| 1595 | } |
| 1596 | if err := rs.steady(); err != nil { |
| 1597 | return fmt.Errorf("%w (failing fast: %v)", cause, err) |
| 1598 | } |
| 1599 | return nil |
| 1600 | } |
| 1601 | |
| 1602 | // edgeRunState is the edge's NRestarts and ActiveEnterTimestampMonotonic, |
| 1603 | // in that order — the order `show -p` is asked for them. |
| 1604 | type edgeRunState struct{ restarts, activeEnter string } |
| 1605 | |
| 1606 | func (s edgeRunState) String() string { return s.restarts + "/" + s.activeEnter } |
| 1607 | |
| 1608 | // steady is the half of the soak that a comparison cannot make: a restart |
| 1609 | // that happened before the first sample is still a restart. |
| 1610 | func (s edgeRunState) steady() error { |
| 1611 | if s.restarts != "0" { |
| 1612 | return fmt.Errorf("edge has crash-restarted since the install (NRestarts=%s)", s.restarts) |
| 1613 | } |
| 1614 | return nil |
| 1615 | } |
| 1616 | |
| 1617 | func (a *systemApplier) edgeRunState() (edgeRunState, error) { |
| 1618 | out, err := a.ops.systemctl("show", "-p", "NRestarts", "-p", "ActiveEnterTimestampMonotonic", "--value", systemEdgeUnit) |
| 1619 | if err != nil { |
| 1620 | return edgeRunState{}, err |
| 1621 | } |
| 1622 | restarts, rest, _ := strings.Cut(strings.TrimSpace(out), "\n") |
| 1623 | return edgeRunState{restarts: strings.TrimSpace(restarts), activeEnter: strings.TrimSpace(rest)}, nil |
| 1624 | } |
| 1625 | |
| 1626 | func (a *systemApplier) gateOnce(ctx context.Context, target string) error { |
| 1627 | _, err := a.gateCheck(ctx, target) |
| 1628 | return err |
| 1629 | } |
| 1630 | |
| 1631 | // gateCheck is gateOnce that also returns what is-active said about the |
| 1632 | // edge, trimmed, for gateFailFast. The checks, their order and their |
| 1633 | // errors are gateOnce's. |
| 1634 | func (a *systemApplier) gateCheck(ctx context.Context, target string) (string, error) { |
| 1635 | out, err := a.ops.systemctl("is-active", systemEdgeUnit) |
| 1636 | state := strings.TrimSpace(out) |
| 1637 | if err != nil || state != "active" { |
| 1638 | if err != nil { |
| 1639 | return state, fmt.Errorf("%s is %s: %w", systemEdgeUnit, state, err) |
| 1640 | } |
| 1641 | return state, fmt.Errorf("%s is %s", systemEdgeUnit, state) |
| 1642 | } |
| 1643 | if err := a.ops.probeEdge(); err != nil { |
| 1644 | return state, fmt.Errorf("edge probe: %w", err) |
| 1645 | } |
| 1646 | if got := a.onDiskBuild(ctx); got != target { |
| 1647 | return state, fmt.Errorf("installed binary reports %q, not %s", got, target) |
| 1648 | } |
| 1649 | return state, nil |
| 1650 | } |
| 1651 | |
| 1652 | // arm enables every unit in the plan and starts the ones the plan says to |
| 1653 | // start: the unit types whose enable-without-start is a silent no-op until |
| 1654 | // the next boot, and of the services only carlos-relay, whose |
| 1655 | // ConditionPathExists makes the start a no-op on a box with no region master |
| 1656 | // (relay spec §3). |
| 1657 | // |
| 1658 | // It never starts the edge, which is running this very code — that is why |
| 1659 | // apply can call it from inside itself. Before #118 that was structural: the |
| 1660 | // applier derived the start list itself and could not name a service other |
| 1661 | // than the relay. Now the list comes from the target build, so the guarantee |
| 1662 | // is enforced here instead, explicitly. |
| 1663 | func (a *systemApplier) arm(plan bootstrapPlan) error { |
| 1664 | // plan.Armed is the TARGET build's answer to "and which of these get a |
| 1665 | // start" (discussion #118). Preferring it is what lets a build introduce |
| 1666 | // a unit that must be started at apply time rather than at the next |
| 1667 | // boot; falling back to the local rule keeps a plan.json from an older |
| 1668 | // build — which is what a rollback stages — working unchanged. |
| 1669 | armed := plan.Armed |
| 1670 | if armed == nil { |
| 1671 | armed = armedUnits(plan.Enabled) |
| 1672 | } |
| 1673 | // Two guards on a list that now comes from another build. The edge is |
| 1674 | // never started here, whatever the plan says. And a unit that is not in |
| 1675 | // Enabled is dropped rather than passed to systemctl: `start` is ONE |
| 1676 | // batched call, so a single name this box has no unit file for would |
| 1677 | // fail the whole call and leave nothing armed at all. |
| 1678 | enabled := make(map[string]bool, len(plan.Enabled)) |
| 1679 | for _, u := range plan.Enabled { |
| 1680 | enabled[u] = true |
| 1681 | } |
| 1682 | kept := armed[:0:0] |
| 1683 | for _, u := range armed { |
| 1684 | switch { |
| 1685 | case u == systemEdgeUnit || u == systemEdgeUnit+".service": |
| 1686 | a.ops.logf("system apply: arm: refusing to start %s, which is running this code", u) |
| 1687 | case !enabled[u]: |
| 1688 | a.ops.logf("system apply: arm: %s is armed by the plan but not enabled by it; skipping", u) |
| 1689 | default: |
| 1690 | kept = append(kept, u) |
| 1691 | } |
| 1692 | } |
| 1693 | for _, args := range systemctlPlanArming(plan.Enabled, kept) { |
| 1694 | if _, err := a.ops.systemctl(args...); err != nil { |
| 1695 | return fmt.Errorf("systemctl %s: %w", strings.Join(args, " "), err) |
| 1696 | } |
| 1697 | } |
| 1698 | return nil |
| 1699 | } |
| 1700 | |
| 1701 | // rollback is step 13: back to carlos.prev and its units, restart, gate, |
| 1702 | // arm the previous plan. applied.json is untouched — the .prev build is |
| 1703 | // what passed a gate, and it is what applied.json already names. |
| 1704 | // |
| 1705 | // A rollback that cannot finish leaves the journal open: the box is then |
| 1706 | // between two builds — the previous binary under the failed build's units, |
| 1707 | // say — and only another recovery puts it right. The one exception is a |
| 1708 | // complete restore whose GATE failed: the files are all the previous |
| 1709 | // build's, restoring them again would change nothing, and the box needs a |
| 1710 | // human rather than a loop. |
| 1711 | func (a *systemApplier) rollback(ctx context.Context, req systemRequest, reason string, j *systemJournal) error { |
| 1712 | a.ops.logf("system apply: %s %s: rolling back: %s", a.channel, j.Target, reason) |
| 1713 | failed := func(what string, err error) error { |
| 1714 | _ = a.writeResult(req, systemOutcomeRollbackFailed, reason+"; "+what+": "+err.Error()) |
| 1715 | return fmt.Errorf("system apply: rollback of %s failed: %s: %v", j.Target, what, err) |
| 1716 | } |
| 1717 | plan, err := a.restoreFiles(ctx, j) |
| 1718 | if err != nil { |
| 1719 | return failed("restore", err) // journal kept: the restore is half done |
| 1720 | } |
| 1721 | // carlos.prev, now on disk: what this rollback's journal finish makes |
| 1722 | // current, checked again under activationGuard at that finish. |
| 1723 | restored, err := a.restoreVersion() |
| 1724 | if err != nil { |
| 1725 | return failed("identity", err) |
| 1726 | } |
| 1727 | // The previous binary is on disk, so the control service goes back |
| 1728 | // too, before the edge, as on the way forward. Logged, not fatal: a |
| 1729 | // rollback's job is a good edge, and a control service that would |
| 1730 | // not restart is caught by the agent's readiness check (it refuses |
| 1731 | // nested wakes until the service runs the installed binary). |
| 1732 | if err := a.restartInstanceControl(); err != nil { |
| 1733 | a.ops.logf("system apply: rollback: instance-control restart: %v (nested wakes refuse until it is restarted)", err) |
| 1734 | } |
| 1735 | if err := a.restartEdge(); err != nil { |
| 1736 | return failed("restart", err) // journal kept: the edge is still to come back |
| 1737 | } |
| 1738 | a.restartRelay() |
| 1739 | if err := a.gate(ctx, j.PrevBuild); err != nil { |
| 1740 | err := failed("previous build failed its gate too", err) |
| 1741 | // The journal finish, under activationGuard (moves floor and hold |
| 1742 | // rule first), closed through closeGuardRollback (the claim release |
| 1743 | // floor's completion of a rolled-back transaction) inside it. |
| 1744 | var closeErr error |
| 1745 | if gerr := a.activationGuard(ctx, restored, restored, func(ctx context.Context) error { |
| 1746 | closeErr = a.closeGuardRollback(ctx) |
| 1747 | return nil |
| 1748 | }); gerr != nil { |
| 1749 | return fmt.Errorf("%v; the journal stays: %v", err, gerr) |
| 1750 | } |
| 1751 | if closeErr != nil { |
| 1752 | return errors.Join(err, closeErr) |
| 1753 | } |
| 1754 | return err |
| 1755 | } |
| 1756 | if err := a.arm(plan); err != nil { |
| 1757 | a.ops.logf("system apply: arm previous plan: %v", err) |
| 1758 | } |
| 1759 | // closeGuardRollback's own error is main's: returned as it is, not as a |
| 1760 | // rollback failure (the journal is already gone by then, unless the |
| 1761 | // armed floor refused to record the rollback, which keeps it). |
| 1762 | var closeErr error |
| 1763 | if err := a.activationGuard(ctx, restored, restored, func(ctx context.Context) error { |
| 1764 | if err := a.writeResult(req, systemOutcomeRolledBack, reason); err != nil { |
| 1765 | return err |
| 1766 | } |
| 1767 | closeErr = a.closeGuardRollback(ctx) |
| 1768 | return nil |
| 1769 | }); err != nil { |
| 1770 | return failed("finish", err) // journal kept |
| 1771 | } |
| 1772 | if closeErr != nil { |
| 1773 | return closeErr |
| 1774 | } |
| 1775 | return fmt.Errorf("system apply: %s rolled back: %s", j.Target, reason) |
| 1776 | } |
| 1777 | |
| 1778 | // restoreFiles is the files-only half shared by rollback and recovery: |
| 1779 | // carlos.prev over carlos (keeping .prev), units_added disabled and |
| 1780 | // removed, the previous unit set copied back (bringing units_removed back), |
| 1781 | // sysusers, tmpfiles, daemon-reload, and `enable` of the previous plan so |
| 1782 | // it comes up at the next boot. No restart, no start: what to do about the |
| 1783 | // running edge is the caller's, and at boot the answer is nothing. |
| 1784 | func (a *systemApplier) restoreFiles(ctx context.Context, j *systemJournal) (bootstrapPlan, error) { |
| 1785 | prev := systemPrevPath(a.optDir) |
| 1786 | if err := a.guardImage(prev); err != nil { |
| 1787 | return bootstrapPlan{}, err |
| 1788 | } |
| 1789 | if _, err := os.Stat(prev); err != nil { |
| 1790 | return bootstrapPlan{}, fmt.Errorf("no carlos.prev: %w", err) |
| 1791 | } |
| 1792 | // The moves floor and the hold rule (moves spec §11), before anything |
| 1793 | // on the box changes, for the version carlos.prev's sidecar records (no |
| 1794 | // sidecar is below any armed floor). A refusal keeps the journal: the |
| 1795 | // box stays on the build it is on, and the caller reports it. |
| 1796 | prevVersion, err := a.restoreVersion() |
| 1797 | if err != nil { |
| 1798 | return bootstrapPlan{}, err |
| 1799 | } |
| 1800 | running := a.onDiskBuild(ctx) |
| 1801 | if _, required, err := a.rootMoves.read(); err != nil { |
| 1802 | return bootstrapPlan{}, err |
| 1803 | } else if required { |
| 1804 | if current, err := readRootFloorBinary(a.optDir, systemBinaryPath(a.optDir)); err == nil { |
| 1805 | running = current.Version |
| 1806 | } |
| 1807 | } |
| 1808 | if err := a.activationRule(prevVersion, func() string { return running }); err != nil { |
| 1809 | return bootstrapPlan{}, err |
| 1810 | } |
| 1811 | // The same quiesce the install did: a root pass that is mid-flight |
| 1812 | // against the failed build must not run on across the swap back. |
| 1813 | prior := a.quiesce() |
| 1814 | // carlos.prev is copied beside the binary first, outside any lock, and |
| 1815 | // renamed over it under the activation guard, so the rule's inputs are |
| 1816 | // held still for a rename rather than a whole binary copy. |
| 1817 | restore := systemBinaryPath(a.optDir) + ".restore" |
| 1818 | if err := copyFileDurable(prev, restore, 0o755); err != nil { |
| 1819 | return bootstrapPlan{}, err |
| 1820 | } |
| 1821 | renamed := false |
| 1822 | if err := a.activationGuard(ctx, prevVersion, running, func(ctx context.Context) error { |
| 1823 | if err := a.rootProtocolAllowsBinary(restore); err != nil { |
| 1824 | return err |
| 1825 | } |
| 1826 | renamed = true |
| 1827 | if err := os.Rename(restore, systemBinaryPath(a.optDir)); err != nil { |
| 1828 | return err |
| 1829 | } |
| 1830 | return syncDir(a.optDir) |
| 1831 | }); err != nil { |
| 1832 | os.Remove(restore) |
| 1833 | // Stopped before the rename (refused or deferred: the floor or a hold |
| 1834 | // changed during the copy, or a writer holds the floor lock): nothing |
| 1835 | // was restored, so the paused triggers go back to how they were. If |
| 1836 | // they cannot be started, they go to rearm-pending.json, captured as |
| 1837 | // they were before the quiesce, and every run starts them first, |
| 1838 | // before any early refusal. The caller keeps the journal. |
| 1839 | if !renamed { |
| 1840 | if rerr := a.rearm(prior); rerr != nil { |
| 1841 | if nerr := a.notePendingRearm(prior); nerr != nil { |
| 1842 | err = fmt.Errorf("%w; the root passes paused for the restore could not be started again: %v, and the retry could not be recorded: %v", err, rerr, nerr) |
| 1843 | } else { |
| 1844 | err = fmt.Errorf("%w; the root passes paused for the restore could not be started again: %v", err, rerr) |
| 1845 | } |
| 1846 | } |
| 1847 | } |
| 1848 | return bootstrapPlan{}, err |
| 1849 | } |
| 1850 | // The unit set to put back is whatever the binary being restored ships, |
| 1851 | // so it comes from carlos.prev and never from the journal. The install |
| 1852 | // already staged it, and that staging is preferred: re-running a |
| 1853 | // bootstrap on a box that is mid-rollback is one more thing that can |
| 1854 | // fail, and it would wipe the good units before it tried. |
| 1855 | prevDir := filepath.Join(a.stagingDir(), "prev") |
| 1856 | plan, err := stagedPlan(prevDir) |
| 1857 | if err != nil { |
| 1858 | if plan, _, err = a.stagePlan(ctx, prev, prevDir); err != nil { |
| 1859 | return bootstrapPlan{}, fmt.Errorf("carlos.prev bootstrap: %w", err) |
| 1860 | } |
| 1861 | } |
| 1862 | if err := a.installUnits(ctx, prevDir, j.UnitsAdded); err != nil { |
| 1863 | return bootstrapPlan{}, err |
| 1864 | } |
| 1865 | if len(plan.Enabled) > 0 { |
| 1866 | if _, err := a.ops.systemctl(append([]string{"enable"}, plan.Enabled...)...); err != nil { |
| 1867 | return bootstrapPlan{}, err |
| 1868 | } |
| 1869 | } |
| 1870 | return plan, nil |
| 1871 | } |
| 1872 | |
| 1873 | // stagedPlan reads back what a stagePlan into dir left there: the plan the |
| 1874 | // binary printed and the units it wrote. An empty unit directory or a |
| 1875 | // missing plan.json means the staging is not there to be reused. |
| 1876 | func stagedPlan(dir string) (bootstrapPlan, error) { |
| 1877 | entries, err := os.ReadDir(filepath.Join(dir, "etc", "systemd", "system")) |
| 1878 | if err != nil { |
| 1879 | return bootstrapPlan{}, err |
| 1880 | } |
| 1881 | if len(entries) == 0 { |
| 1882 | return bootstrapPlan{}, fmt.Errorf("no units staged in %s", dir) |
| 1883 | } |
| 1884 | var plan bootstrapPlan |
| 1885 | if err := readJSONFile(filepath.Join(dir, "plan.json"), &plan); err != nil { |
| 1886 | return bootstrapPlan{}, err |
| 1887 | } |
| 1888 | return plan, nil |
| 1889 | } |
| 1890 | |
| 1891 | // unitsMinus is have − drop, order preserved. |
| 1892 | func unitsMinus(have, drop []string) []string { |
| 1893 | skip := map[string]bool{} |
| 1894 | for _, d := range drop { |
| 1895 | skip[d] = true |
| 1896 | } |
| 1897 | var out []string |
| 1898 | for _, h := range have { |
| 1899 | if !skip[h] { |
| 1900 | out = append(out, h) |
| 1901 | } |
| 1902 | } |
| 1903 | return out |
| 1904 | } |
| 1905 | |
| 1906 | // copyFileDurable is writeFileDurable for a file too large to hold in |
| 1907 | // memory — the binary itself — and lands the same way. |
| 1908 | func copyFileDurable(src, dst string, mode os.FileMode) error { |
| 1909 | in, err := os.Open(src) |
| 1910 | if err != nil { |
| 1911 | return err |
| 1912 | } |
| 1913 | defer in.Close() |
| 1914 | return writeDurable(dst, mode, func(w io.Writer) error { |
| 1915 | _, err := io.Copy(w, in) |
| 1916 | return err |
| 1917 | }) |
| 1918 | } |
| 1919 | |
| 1920 | // recoverIfNeeded is step 0: an open journal means a previous run was cut |
| 1921 | // off, and nothing else happens until the box is whole again. Before the |
| 1922 | // gate → back, files only, restarting the edge only when the edge has |
| 1923 | // already started this boot (at boot, systemd is about to start it and a |
| 1924 | // restart from an ordering-before unit would deadlock). At gated → forward: |
| 1925 | // that build proved itself, and undoing it would throw away a good apply. |
| 1926 | // The journal is removed LAST, after the result is written and the restart |
| 1927 | // issued, so a recovery that dies partway is recovered again. |
| 1928 | // |
| 1929 | // It reports whether it moved the box: a run that recovered has written the |
| 1930 | // result it answers with, and does not then go on to apply a request. |
| 1931 | func (a *systemApplier) recoverIfNeeded(ctx context.Context) (bool, error) { |
| 1932 | j, err := readSystemJournal(a.optDir) |
| 1933 | if err != nil || j == nil { |
| 1934 | return false, err |
| 1935 | } |
| 1936 | // The result belongs to the promote being recovered, which the journal |
| 1937 | // names in full. The request is used only when it is that same promote, |
| 1938 | // version and promoted_at both: a re-promote of the same version is a |
| 1939 | // new tuple, and recording this rollback against it would refuse the |
| 1940 | // operator's retry before it started. |
| 1941 | req, _ := a.readRequest() |
| 1942 | if req.Version != j.Target || !req.PromotedAt.Equal(j.promoted()) { |
| 1943 | req = systemRequest{Version: j.Target, PromotedAt: j.promoted()} |
| 1944 | } |
| 1945 | a.ops.logf("system apply: recovering an apply of %s interrupted at %s", j.Target, j.Phase) |
| 1946 | if j.Phase == systemPhaseBackingUp { |
| 1947 | // A legacy journal: this version no longer writes one for the |
| 1948 | // backup (see decidePrev), but a box killed mid-apply by the |
| 1949 | // version that did comes up running this one. Nothing on the box |
| 1950 | // had changed yet — only a half-written .prev copy can be left, |
| 1951 | // and it is not the file anything reads. The run carries on to the |
| 1952 | // request it was started for. |
| 1953 | os.Remove(systemPrevPath(a.optDir) + ".tmp") |
| 1954 | return false, a.closeGuardJournal(ctx) |
| 1955 | } |
| 1956 | // Forward from gated — but only while the build that passed the gate is |
| 1957 | // still the one on disk. A binary that has moved since means this is a |
| 1958 | // pre-gate state whatever the journal says. |
| 1959 | if j.Phase == systemPhaseGated { |
| 1960 | if got := a.onDiskBuild(ctx); got != j.Target { |
| 1961 | a.ops.logf("system apply: recovery: %s passed its gate but the binary on disk reports %q; rolling back instead", j.Target, got) |
| 1962 | } else if err := a.floorAllowsForward(j.Target); err != nil { |
| 1963 | // A floor raised since the gate (moves spec §11; fix round 1): the |
| 1964 | // build is not finished forward. Back to carlos.prev, which |
| 1965 | // restoreFiles judges in turn; if that is refused too, the |
| 1966 | // journal stays and the box stays as it is. |
| 1967 | a.ops.logf("system apply: recovery: %s passed its gate but %v; rolling back instead", j.Target, err) |
| 1968 | } else { |
| 1969 | plan, _, err := a.stagePlan(ctx, systemBinaryPath(a.optDir), filepath.Join(a.stagingDir(), "new")) |
| 1970 | if err != nil { |
| 1971 | return false, fmt.Errorf("system apply: recovery: %w", err) |
| 1972 | } |
| 1973 | // Finishing the journal is the forward path's activation: the |
| 1974 | // rule again, under the guard, at the step itself, since the |
| 1975 | // floor can rise while the plan is staged (fix round 2). |
| 1976 | finished := false |
| 1977 | err = a.activationGuard(ctx, j.Target, j.Target, func(ctx context.Context) error { |
| 1978 | if err := a.rootProtocolAllowsBinary(systemBinaryPath(a.optDir)); err != nil { |
| 1979 | return err |
| 1980 | } |
| 1981 | finished = true |
| 1982 | return a.finish(ctx, req, j.Target, plan) |
| 1983 | }) |
| 1984 | if finished { |
| 1985 | return true, err |
| 1986 | } |
| 1987 | var deferred activationDeferred |
| 1988 | if errors.As(err, &deferred) { |
| 1989 | return false, fmt.Errorf("system apply: recovery: %w", err) // journal kept |
| 1990 | } |
| 1991 | a.ops.logf("system apply: recovery: %s passed its gate but %v; rolling back instead", j.Target, err) |
| 1992 | } |
| 1993 | } |
| 1994 | plan, err := a.restoreFiles(ctx, j) |
| 1995 | if err != nil { |
| 1996 | return false, fmt.Errorf("system apply: recovery: %w", err) // journal kept: the next run recovers again |
| 1997 | } |
| 1998 | // carlos.prev, now on disk: what the finish below makes current. |
| 1999 | restored, err := a.restoreVersion() |
| 2000 | if err != nil { |
| 2001 | return false, err |
| 2002 | } |
| 2003 | if a.ops.edgeStartedThisBoot() { |
| 2004 | // The control service first, as in install and rollback. Not at |
| 2005 | // boot: it is socket-activated and not yet running, and started |
| 2006 | // before the edge it would run its probe against an edge whose |
| 2007 | // cgroup does not exist yet and record a failing verdict for the |
| 2008 | // next five minutes. Blocking is safe: nothing orders this unit |
| 2009 | // against carlos-system-recover.service. |
| 2010 | if err := a.restartInstanceControl(); err != nil { |
| 2011 | a.ops.logf("system apply: recovery: instance-control restart: %v (nested wakes refuse until it is restarted)", err) |
| 2012 | } |
| 2013 | a.ops.systemctl("reset-failed", systemEdgeUnit) |
| 2014 | // --no-block: this can be running as the edge's own OnFailure= |
| 2015 | // handler, and a blocking restart of the unit that triggered it |
| 2016 | // would wait on the job that is waiting on this one. |
| 2017 | if _, err := a.ops.systemctl("restart", "--no-block", systemEdgeUnit); err != nil { |
| 2018 | return false, fmt.Errorf("system apply: recovery: restart: %w", err) |
| 2019 | } |
| 2020 | var armed []string |
| 2021 | for _, u := range plan.Enabled { |
| 2022 | if strings.HasSuffix(u, ".timer") || strings.HasSuffix(u, ".path") { |
| 2023 | armed = append(armed, u) |
| 2024 | } |
| 2025 | } |
| 2026 | if len(armed) > 0 { |
| 2027 | a.ops.systemctl(append([]string{"start"}, armed...)...) |
| 2028 | } |
| 2029 | } |
| 2030 | // The backward finish is guarded like every other: a floor raised since |
| 2031 | // the restore's rename keeps the journal, and the next run judges again. |
| 2032 | // closeGuardRollback's own error is main's, returned as main returns it. |
| 2033 | var closeErr error |
| 2034 | if err := a.activationGuard(ctx, restored, restored, func(ctx context.Context) error { |
| 2035 | if err := a.writeResult(req, systemOutcomeRolledBack, "interrupted at "+j.Phase); err != nil { |
| 2036 | return err |
| 2037 | } |
| 2038 | closeErr = a.closeGuardRollback(ctx) |
| 2039 | return nil |
| 2040 | }); err != nil { |
| 2041 | return false, fmt.Errorf("system apply: recovery: %w", err) // journal kept |
| 2042 | } |
| 2043 | return true, closeErr |
| 2044 | } |
| 2045 | |
| 2046 | // record turns a refusal or deferral into result.json and a nil return. |
| 2047 | func (a *systemApplier) record(req systemRequest, err error) error { |
| 2048 | outcome := systemOutcomeRefused |
| 2049 | if errors.Is(err, errSystemDeferred) { |
| 2050 | outcome = systemOutcomeDeferred |
| 2051 | } |
| 2052 | a.ops.logf("system apply: %s %s: %s: %v", a.channel, req.Version, outcome, err) |
| 2053 | return a.writeResult(req, outcome, err.Error()) |
| 2054 | } |
| 2055 | |
| 2056 | func (a *systemApplier) writeResult(req systemRequest, outcome, reason string) error { |
| 2057 | return writeJSONDurable(a.file("result.json"), systemResult{ |
| 2058 | Channel: a.channel, Version: req.Version, PromotedAt: req.PromotedAt, |
| 2059 | Outcome: outcome, Reason: reason, At: a.ops.now().UTC(), |
| 2060 | }, 0o644) |
| 2061 | } |
| 2062 | |
| 2063 | // readRequest is the one read under /data: a fixed path reached without |
| 2064 | // following a single symlink, a regular file under 4 KiB, a version that |
| 2065 | // passes validSystemVersion. Anything else is an error, and root acts on |
| 2066 | // nothing. |
| 2067 | func (a *systemApplier) readRequest() (systemRequest, error) { |
| 2068 | f, err := openBeneath(a.dataDir, "system", systemComponent, "requested.json") |
| 2069 | if err != nil { |
| 2070 | return systemRequest{}, err |
| 2071 | } |
| 2072 | defer f.Close() |
| 2073 | st, err := f.Stat() |
| 2074 | if err != nil { |
| 2075 | return systemRequest{}, err |
| 2076 | } |
| 2077 | // openBeneath already checked the inode; bound the request size too. |
| 2078 | if !st.Mode().IsRegular() || st.Size() > systemMaxRequest { |
| 2079 | return systemRequest{}, fmt.Errorf("request: not a regular file under %d bytes", systemMaxRequest) |
| 2080 | } |
| 2081 | var req systemRequest |
| 2082 | if err := json.NewDecoder(io.LimitReader(f, systemMaxRequest)).Decode(&req); err != nil { |
| 2083 | return systemRequest{}, fmt.Errorf("request: %w", err) |
| 2084 | } |
| 2085 | if !validSystemVersion(req.Version) { |
| 2086 | return systemRequest{}, fmt.Errorf("request: version %q is not a version", req.Version) |
| 2087 | } |
| 2088 | if req.PromotedAt.IsZero() { |
| 2089 | return systemRequest{}, errors.New("request: promoted_at unset") |
| 2090 | } |
| 2091 | return req, nil |
| 2092 | } |
| 2093 | |
| 2094 | // verifyPointer fetches THIS box's channel pointer and verifies it with |
| 2095 | // the release public key. The channel is root's own (host.env), never the |
| 2096 | // request's. A store error is deferred, a bad pointer refused. |
| 2097 | // |
| 2098 | // This is where the store is opened and the pubkey parsed — a store the box |
| 2099 | // cannot reach, or an unset CARLOS_RELEASE_PUBKEY, is deferred like any |
| 2100 | // other "could not read the pointer", recorded in result.json where an |
| 2101 | // operator sees it, rather than exiting before the run began. |
| 2102 | func (a *systemApplier) verifyPointer(ctx context.Context) (*release.SystemPointer, error) { |
| 2103 | store, pub, err := a.openStore(ctx) |
| 2104 | if err != nil { |
| 2105 | return nil, fmt.Errorf("%w: %v", errSystemDeferred, err) |
| 2106 | } |
| 2107 | a.store = store |
| 2108 | p, _, err := release.GetSystemPointer(ctx, store, pub, release.SystemComponentRouter, a.channel) |
| 2109 | switch { |
| 2110 | case err == nil: |
| 2111 | if _, required, e := a.rootMoves.read(); e != nil { |
| 2112 | return nil, e |
| 2113 | } else if required { |
| 2114 | if e := rememberRootFloorSystemIdentity(ctx, store, pub, a.optDir, p); e != nil { |
| 2115 | return nil, e |
| 2116 | } |
| 2117 | } |
| 2118 | return p, nil |
| 2119 | case errors.Is(err, os.ErrNotExist): |
| 2120 | return nil, fmt.Errorf("%w: no %s pointer for this box's channel", errSystemRefused, a.channel) |
| 2121 | case errors.Is(err, release.ErrBadSystemPointer): |
| 2122 | return nil, fmt.Errorf("%w: %v", errSystemRefused, err) |
| 2123 | default: |
| 2124 | return nil, fmt.Errorf("%w: %v", errSystemDeferred, err) |
| 2125 | } |
| 2126 | } |
| 2127 | |
| 2128 | // onDiskBuild asks the installed binary what it is, with a deadline. Not |
| 2129 | // version.Build: after an interrupted apply the two can differ. |
| 2130 | func (a *systemApplier) onDiskBuild(ctx context.Context) string { |
| 2131 | out, err := a.guardRun(ctx, systemVersionWait, systemBinaryPath(a.optDir), "version") |
| 2132 | if err != nil { |
| 2133 | return "" |
| 2134 | } |
| 2135 | return carlosVersionOf(out) |
| 2136 | } |
| 2137 | |
| 2138 | // carlosVersionOf reads `carlos version` output — exactly "carlos <build>", |
| 2139 | // nothing else. A build that merely starts with the expected version, or |
| 2140 | // output with anything else on the line, yields "" and matches nothing. |
| 2141 | func carlosVersionOf(out string) string { |
| 2142 | v, ok := strings.CutPrefix(strings.TrimSpace(out), "carlos ") |
| 2143 | if !ok { |
| 2144 | return "" |
| 2145 | } |
| 2146 | return v |
| 2147 | } |
| 2148 | |
| 2149 | // stageCandidate copies the agent's cache into staging/ (0700) as 0600, |
| 2150 | // verifies the copy's sha against the pointer, makes it executable, and |
| 2151 | // runs its `version`. Nothing under /data is read after the copy. |
| 2152 | func (a *systemApplier) stageCandidate(ctx context.Context, p *release.SystemPointer, req systemRequest) (string, error) { |
| 2153 | art, ok := p.Current.Artifacts[a.platform] |
| 2154 | if !ok { |
| 2155 | return "", fmt.Errorf("pointer names no artifact for %s", a.platform) |
| 2156 | } |
| 2157 | os.RemoveAll(a.stagingDir()) |
| 2158 | if err := os.MkdirAll(a.stagingDir(), 0o700); err != nil { |
| 2159 | return "", err |
| 2160 | } |
| 2161 | if err := os.Chmod(a.stagingDir(), 0o700); err != nil { |
| 2162 | return "", err |
| 2163 | } |
| 2164 | src, err := openBeneath(a.dataDir, systemCacheParts(req.Version)...) |
| 2165 | if err != nil { |
| 2166 | return "", fmt.Errorf("candidate: %w", err) |
| 2167 | } |
| 2168 | defer src.Close() |
| 2169 | // openBeneath checked the inode before opening it for I/O. |
| 2170 | if st, err := src.Stat(); err != nil || !st.Mode().IsRegular() { |
| 2171 | return "", errors.New("candidate: not a regular file") |
| 2172 | } |
| 2173 | staged := filepath.Join(a.stagingDir(), "carlos") |
| 2174 | dst, err := os.OpenFile(staged, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) |
| 2175 | if err != nil { |
| 2176 | return "", err |
| 2177 | } |
| 2178 | h := sha256.New() |
| 2179 | n, err := io.Copy(io.MultiWriter(dst, h), io.LimitReader(src, systemArtifactLimit+1)) |
| 2180 | if cerr := dst.Close(); err == nil { |
| 2181 | err = cerr // a close that fails is a copy that did not land |
| 2182 | } |
| 2183 | if err != nil { |
| 2184 | return "", fmt.Errorf("candidate: copying into staging: %w", err) |
| 2185 | } |
| 2186 | if n > systemArtifactLimit { |
| 2187 | return "", fmt.Errorf("candidate: larger than %d bytes", systemArtifactLimit) |
| 2188 | } |
| 2189 | if got := hex.EncodeToString(h.Sum(nil)); got != art.SHA256 { |
| 2190 | return "", fmt.Errorf("candidate: sha256 %s does not match the signed %s", got, art.SHA256) |
| 2191 | } |
| 2192 | if err := os.Chmod(staged, 0o755); err != nil { |
| 2193 | return "", err |
| 2194 | } |
| 2195 | out, err := a.guardRun(ctx, systemVersionWait, staged, "version") |
| 2196 | if err != nil { |
| 2197 | return "", fmt.Errorf("candidate: running its version: %w", err) |
| 2198 | } |
| 2199 | if carlosVersionOf(out) != p.Current.Version { |
| 2200 | return "", fmt.Errorf("candidate reports %q, not %s", strings.TrimSpace(out), p.Current.Version) |
| 2201 | } |
| 2202 | return staged, nil |
| 2203 | } |
| 2204 | |
| 2205 | func (a *systemApplier) restoreVersion() (string, error) { |
| 2206 | if _, required, err := a.rootMoves.read(); err != nil { |
| 2207 | return "", err |
| 2208 | } else if required { |
| 2209 | identity, err := readRootFloorBinary(a.optDir, systemPrevPath(a.optDir)) |
| 2210 | return identity.Version, err |
| 2211 | } |
| 2212 | version, _ := readLine(prevVersionPath(a.optDir)) |
| 2213 | return version, nil |
| 2214 | } |
| 2215 | |
amadan