| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/rand" |
| 6 | "encoding/hex" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "golang.org/x/sys/unix" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "strconv" |
| 13 | "strings" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/hostbox" |
| 17 | "github.com/carlosframework/platform/internal/hostguard" |
| 18 | "github.com/carlosframework/platform/internal/hostops" |
| 19 | "github.com/carlosframework/platform/internal/release" |
| 20 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 21 | "github.com/carlosframework/platform/internal/rootstate" |
| 22 | "github.com/carlosframework/platform/internal/safefile" |
| 23 | ) |
| 24 | |
| 25 | var rootMigrationQuiesce = quiesceLegacyClaimJournals |
| 26 | |
| 27 | func migrateRootClaimAuthority(ctx context.Context, store release.Store, r ClaimRuntimeResources, owners ClaimOwnerBindings, actor hostbox.Actor, journals *hostbox.JournalService, launches *hostbox.LaunchJournal) error { |
| 28 | if err := claimActivationPreflight(r.Registry); err != nil { |
| 29 | return err |
| 30 | } |
| 31 | return withRootReleaseStore(ctx, func() (rootstate.Store, error) { return rootstate.Open(rootReleasePrivate, true, false) }, func() error { |
| 32 | if err := claimActivationVerified(ctx, store, r); err != nil { |
| 33 | return err |
| 34 | } |
| 35 | if err := verifyOfflineRootMigration(ctx); err != nil { |
| 36 | return err |
| 37 | } |
| 38 | state, err := rootstate.Open(rootMigrationDir, true, false) |
| 39 | if err != nil { |
| 40 | return err |
| 41 | } |
| 42 | defer state.Close() |
| 43 | execution, err := rootstate.Open(rootparticipation.Namespace, true, false) |
| 44 | if err != nil { |
| 45 | return err |
| 46 | } |
| 47 | defer execution.Close() |
| 48 | fences, err := openClaimRootAuthority() |
| 49 | if err != nil { |
| 50 | return err |
| 51 | } |
| 52 | defer fences.Close() |
| 53 | root := hostguard.NewRootAuthority(fences) |
| 54 | id := rootMigrationIdentity{Deployment: owners.RetirementAuthority.Directory.Bucket, Box: actor.Box, Enrollment: actor.Enrollment, Generation: 1} |
| 55 | collector := rootMigrationCollector{retirements: productionGenerationRetirements, suspects: func(ctx context.Context) ([]string, error) { return rootMigrationLegacySuspects(ctx, r.DataDir) }, store: store, directory: owners.RetirementAuthority.Directory, identity: id, seeds: r.Registry, installations: root.MigrationInstallations, launches: launches.GenerationLaunches, boot: func() (string, error) { |
| 56 | raw, err := os.ReadFile("/proc/sys/kernel/random/boot_id") |
| 57 | return strings.TrimSpace(string(raw)), err |
| 58 | }, routes: r.Registry.List, dataDir: r.DataDir, reservations: func(ctx context.Context) ([]release.GenerationReservation, error) { |
| 59 | state, err := rootstate.Open(release.RootGenerationsDir, false, false) |
| 60 | if errors.Is(err, os.ErrNotExist) { |
| 61 | return nil, nil |
| 62 | } |
| 63 | if err != nil { |
| 64 | return nil, err |
| 65 | } |
| 66 | defer state.Close() |
| 67 | return (&release.RootGenerations{Store: state}).Reservations(ctx) |
| 68 | }} |
| 69 | roots := map[string]string{"data": r.DataDir, "opt": r.OptDir, "units": r.UnitsDir} |
| 70 | guard := hostguard.Guard{Root: root, DataDir: r.DataDir, Claims: hostguard.StoreReader{Store: store}} |
| 71 | // Reads go through the same typed transport used by the running controller. |
| 72 | _, inventory := hostbox.RemoteJournals(func(ctx context.Context, q hostbox.JournalRequest) (hostbox.JournalResponse, error) { |
| 73 | return journals.Execute(ctx, "root-migration", q) |
| 74 | }) |
| 75 | timers := rootMigrationTimers{state: state, registry: r.Registry, inventory: inventory, guard: guard} |
| 76 | completeInventory := func(ctx context.Context, item rootMigrationItem, target hostguard.FenceTarget) error { |
| 77 | observed, err := (hostguard.StoreReader{Store: store}).Read(ctx, item.Installation.Host) |
| 78 | if err != nil || observed.Claim == nil { |
| 79 | return errRootMigrationBusy |
| 80 | } |
| 81 | canonical, _, err := rootClaimRecord(ctx, store, observed.Claim, r.DataDir) |
| 82 | if err != nil { |
| 83 | return err |
| 84 | } |
| 85 | return timers.adopt(ctx, item, target, canonical) |
| 86 | } |
| 87 | coordinator := ClaimStopCoordinator{Guard: guard, Claims: hostguard.StoreReader{Store: store}, Inventory: inventory, Kernel: newProductionStopKernel(launches, roots), Evidence: store, EvidenceSigner: owners.EvidenceKey, RequestLimit: hostbox.RequestLimit, Now: time.Now, NewID: func() string { return "migration-" + randomMigrationID() }} |
| 88 | m := rootMigration{prepareFloors: func(context.Context) error { |
| 89 | return prepareRootFloorDirs(r.OptDir) |
| 90 | }, guardReady: func(context.Context) error { |
| 91 | return verifyRootMigrationFloors(r.OptDir, rootFloorIdentity{Deployment: id.Deployment, Box: id.Box, Enrollment: id.Enrollment}) |
| 92 | }, state: state, execution: execution, identity: id, quiesce: rootMigrationQuiesce, observe: collector.collect, deny: func(ctx context.Context, item rootMigrationItem) error { |
| 93 | return root.RetainMigrationDeny(ctx, item.Installation, item.Session) |
| 94 | }, prepareRecovery: func(ctx context.Context) error { |
| 95 | // The outer exclusion owns the edge mask through census completion. The |
| 96 | // importer must neither acquire nor release that same mask itself. |
| 97 | return journals.MigrateLegacy(ctx, filepath.Join(r.DataDir, ".hostclaims"), actor.Box, actor.Enrollment, func(context.Context) (func(), error) { return func() {}, nil }, legacyJournalProof(store, owners.RetirementAuthority.Directory), func(context.Context) error { return nil }) |
| 98 | }, recover: func(ctx context.Context, item rootMigrationItem) error { |
| 99 | list, err := launches.GenerationLaunches(ctx) |
| 100 | if err != nil { |
| 101 | return err |
| 102 | } |
| 103 | for _, l := range list { |
| 104 | target := l.Target |
| 105 | in := item.Installation |
| 106 | if target.Host != in.Host || target.Incarnation != in.Incarnation || target.Stamp != in.Stamp || target.Session != item.Session { |
| 107 | continue |
| 108 | } |
| 109 | if pending, exists, err := timers.load(item); err != nil { |
| 110 | return err |
| 111 | } else if exists { |
| 112 | if err := verifyMigrationStopped(ctx, launches, inventory, l, roots); err != nil { |
| 113 | return err |
| 114 | } |
| 115 | return completeInventory(ctx, item, pending.Target) |
| 116 | } |
| 117 | result, err := recoverMigrationStop(ctx, coordinator, store, owners.RetirementAuthority.Directory, launches, inventory, l, item.ClaimETag, roots) |
| 118 | if err != nil { |
| 119 | return err |
| 120 | } |
| 121 | if err := publishMigrationReceipt(ctx, coordinator, result); err != nil { |
| 122 | return err |
| 123 | } |
| 124 | return completeInventory(ctx, item, result.Target) |
| 125 | } |
| 126 | return ErrClaimStopInventory |
| 127 | }, ready: func(ctx context.Context, s rootMigrationSnapshot) error { |
| 128 | if err := verifyRootMigrationRetained(ctx, root, s); err != nil { |
| 129 | return err |
| 130 | } |
| 131 | var all []hostbox.StopInventory |
| 132 | for _, item := range s.Items { |
| 133 | in, revision, err := inventory.Snapshot(ctx, item.Installation.Host, hostguard.Captured{Incarnation: item.Installation.Incarnation, Stamp: item.Installation.Stamp}) |
| 134 | if err != nil { |
| 135 | return err |
| 136 | } |
| 137 | // A missing independently established inventory cannot be filled from the |
| 138 | // agent's markers, even after importing an otherwise valid stop proof. |
| 139 | retained, e := root.Installation(ctx, item.Installation.Host, hostguard.Captured{Incarnation: item.Installation.Incarnation, Stamp: item.Installation.Stamp}) |
| 140 | if e != nil { |
| 141 | return e |
| 142 | } |
| 143 | if revision == 0 || retained.InventoryRevision != revision || !in.Complete { |
| 144 | return ErrClaimStopInventory |
| 145 | } |
| 146 | if err := (ClaimRootHTTP{Reconcile: hostbox.ReconcileAdapter{Registry: r.Registry}}).verifyInventoryTimers(in); err != nil { |
| 147 | return err |
| 148 | } |
| 149 | for _, e := range in.Entries { |
| 150 | if e.Kind == "timer" { |
| 151 | if err := (ClaimRootHTTP{Reconcile: hostbox.ReconcileAdapter{Registry: r.Registry}}).verifyInventoryTimer(in.Host, timers.owner(item, e.RowIdentity), e); err != nil { |
| 152 | return err |
| 153 | } |
| 154 | } |
| 155 | } |
| 156 | all = append(all, in) |
| 157 | } |
| 158 | masked, err := verifyMigrationWindow(r.MigrationWindow, r) |
| 159 | if err != nil { |
| 160 | return err |
| 161 | } |
| 162 | shared, err := claimMigrationSharedPaths(r, masked, s) |
| 163 | if err != nil { |
| 164 | return err |
| 165 | } |
| 166 | if err := hostbox.CheckMigrationRoots(ctx, roots, shared, all); err != nil { |
| 167 | return err |
| 168 | } |
| 169 | if err := migrationExecutionEmpty(execution); err != nil { |
| 170 | return err |
| 171 | } |
| 172 | // The migration's own service-user registry child is still running; |
| 173 | // it is this command's, not a retired generation's. |
| 174 | if _, _, err := productionGenerationKernelCensus(context.WithValue(ctx, generationCensusParentKey{}, os.Getpid())); err != nil { |
| 175 | return err |
| 176 | } |
| 177 | generations, err := rootstate.Open(release.RootGenerationsDir, true, false) |
| 178 | if err != nil { |
| 179 | return err |
| 180 | } |
| 181 | generations.Close() |
| 182 | floorID := rootFloorIdentity{Deployment: id.Deployment, Box: id.Box, Enrollment: id.Enrollment} |
| 183 | if err := prepareRootFloorDirs(r.OptDir); err != nil { |
| 184 | return err |
| 185 | } |
| 186 | for _, dir := range []string{rootFloorDir(r.OptDir), rootSeedsFloorDir(r.OptDir)} { |
| 187 | f, err := openRootProtocolFloors(dir, floorID, true) |
| 188 | if err != nil { |
| 189 | return err |
| 190 | } |
| 191 | f.close() |
| 192 | } |
| 193 | return verifyRootMigrationFloors(r.OptDir, rootFloorIdentity{Deployment: id.Deployment, Box: id.Box, Enrollment: id.Enrollment}) |
| 194 | }, publish: func(ctx context.Context, report hostops.CensusReport) error { |
| 195 | if err := initializeMigrationFailures(r.DataDir, execution); err != nil { |
| 196 | return err |
| 197 | } |
| 198 | return hostops.PublishCensusReport(ctx, store, owners.RetirementAuthority.Directory, report, owners.EvidenceKey) |
| 199 | }} |
| 200 | return m.run(ctx) |
| 201 | }) |
| 202 | } |
| 203 | |
| 204 | // prepareRootFloorDirs makes the two public floor directories, guard/moves |
| 205 | // and guard/seeds, 0755 root:root whatever the operator shell's umask |
| 206 | // (#317's class). rootstate creates a public directory with Mkdirat(0755) |
| 207 | // and never chmods, so under umask 077 it comes out 0700 and rootstate's |
| 208 | // public open refuses it; an interrupted earlier attempt (mkdir done, chmod |
| 209 | // not) leaves the same. The migration runs this under its lock and |
| 210 | // quiesce, before guardReady's first floor read, so a rerun repairs what an |
| 211 | // earlier attempt left instead of refusing it forever; ready runs it again |
| 212 | // before it opens the floors. |
| 213 | // |
| 214 | // Nothing is changed until the path to it has been checked: see |
| 215 | // repairRootFloorDir. |
| 216 | func prepareRootFloorDirs(opt string) error { return prepareRootFloorDirsOwned("/", opt, 0, 0) } |
| 217 | |
| 218 | // prepareRootFloorDirsOwned is prepareRootFloorDirs with the directory the |
| 219 | // walk starts from and the owner every directory on it must have; |
| 220 | // production passes / and root. Unprivileged tests pass a scratch directory |
| 221 | // of their own and their own uid and gid, since the real ancestors of their |
| 222 | // scratch are not theirs. |
| 223 | func prepareRootFloorDirsOwned(base, opt string, uid, gid uint32) error { |
| 224 | for _, dir := range []string{rootFloorDir(opt), rootSeedsFloorDir(opt)} { |
| 225 | if err := repairRootFloorDir(base, dir, uid, gid); err != nil { |
| 226 | return err |
| 227 | } |
| 228 | } |
| 229 | return nil |
| 230 | } |
| 231 | |
| 232 | // repairRootFloorDir walks from base to dir by descriptor (walkNoFollowDir, |
| 233 | // consoleDir's walk) before it changes anything. Every directory from base |
| 234 | // to dir's parent must be a real directory, not a symlink, owned by uid and |
| 235 | // not writable by group or others. Only then is dir made, if it is missing, |
| 236 | // by Mkdirat beneath that checked parent; it is opened with O_NOFOLLOW, must |
| 237 | // be a directory owned by uid and gid, and is Fchmodded to 0755. A symlink |
| 238 | // or a missing directory on the way, an untrusted ancestor, or a dir that is |
| 239 | // a symlink, a file or another owner's refuses, and nothing is made or |
| 240 | // changed. |
| 241 | func repairRootFloorDir(base, dir string, uid, gid uint32) error { |
| 242 | refuse := func(err error) error { return &os.PathError{Op: "repair", Path: dir, Err: err} } |
| 243 | if !filepath.IsAbs(base) || filepath.Clean(base) != base || !filepath.IsAbs(dir) || filepath.Clean(dir) != dir { |
| 244 | return refuse(os.ErrPermission) |
| 245 | } |
| 246 | rel, err := filepath.Rel(base, dir) |
| 247 | if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, "../") { |
| 248 | return refuse(os.ErrPermission) |
| 249 | } |
| 250 | check := func(fd int, last bool) error { |
| 251 | var st unix.Stat_t |
| 252 | if err := unix.Fstat(fd, &st); err != nil { |
| 253 | return err |
| 254 | } |
| 255 | if st.Mode&unix.S_IFMT != unix.S_IFDIR || st.Uid != uid || !last && st.Mode&0o022 != 0 || last && st.Gid != gid { |
| 256 | return os.ErrPermission |
| 257 | } |
| 258 | return nil |
| 259 | } |
| 260 | fd, err := walkNoFollowDir(base, strings.Split(rel, "/"), func(last bool) bool { return last }, check) |
| 261 | if err != nil { |
| 262 | return refuse(err) |
| 263 | } |
| 264 | defer unix.Close(fd) |
| 265 | var st unix.Stat_t |
| 266 | if err := unix.Fstat(fd, &st); err != nil { |
| 267 | return refuse(err) |
| 268 | } |
| 269 | if st.Mode&07777 != 0755 { |
| 270 | if err := unix.Fchmod(fd, 0755); err != nil { |
| 271 | return refuse(err) |
| 272 | } |
| 273 | if err := unix.Fstat(fd, &st); err != nil { |
| 274 | return refuse(err) |
| 275 | } |
| 276 | } |
| 277 | if st.Mode&07777 != 0755 { |
| 278 | return refuse(os.ErrPermission) |
| 279 | } |
| 280 | return nil |
| 281 | } |
| 282 | |
| 283 | func migrationExecutionEmpty(state rootstate.Store) error { |
| 284 | names, err := state.Names() |
| 285 | if err != nil { |
| 286 | return err |
| 287 | } |
| 288 | for _, name := range names { |
| 289 | if name == "census.json" || strings.HasSuffix(name, ".lock") { |
| 290 | continue |
| 291 | } |
| 292 | return errRootMigrationBusy |
| 293 | } |
| 294 | return nil |
| 295 | } |
| 296 | |
| 297 | func randomMigrationID() string { |
| 298 | var b [16]byte |
| 299 | if _, err := rand.Read(b[:]); err != nil { |
| 300 | return "" |
| 301 | } |
| 302 | return hex.EncodeToString(b[:]) |
| 303 | } |
| 304 | func publishMigrationReceipt(ctx context.Context, c ClaimStopCoordinator, result StopResult) error { |
| 305 | var receipt StopReceipt |
| 306 | if json.Unmarshal(result.Bytes, &receipt) != nil { |
| 307 | return ErrClaimStopReceipt |
| 308 | } |
| 309 | result.Key = stopEvidenceKey(receipt.Host, receipt.OperationID, receipt.Box, receipt.ID) |
| 310 | result.Digest = digestRawStop(result.Bytes) |
| 311 | return c.Publish(ctx, result) |
| 312 | } |
| 313 | |
| 314 | func rootMigrationLegacySuspects(ctx context.Context, dd string) ([]string, error) { |
| 315 | dir, err := safefile.OpenDir(filepath.Join(dd, "hostguard")) |
| 316 | if errors.Is(err, os.ErrNotExist) { |
| 317 | return nil, nil |
| 318 | } |
| 319 | if err != nil { |
| 320 | return nil, err |
| 321 | } |
| 322 | defer dir.Close() |
| 323 | names, err := dir.Readdirnames(-1) |
| 324 | if err != nil { |
| 325 | return nil, err |
| 326 | } |
| 327 | out := []string{} |
| 328 | for _, name := range names { |
| 329 | if err := ctx.Err(); err != nil { |
| 330 | return nil, err |
| 331 | } |
| 332 | if strings.HasSuffix(name, ".json") { |
| 333 | out = append(out, strings.TrimSuffix(name, ".json")) |
| 334 | } |
| 335 | } |
| 336 | return out, nil |
| 337 | } |
| 338 | |
| 339 | func quiesceLegacyRootMutators(ctx context.Context) error { |
| 340 | entries, err := os.ReadDir("/proc") |
| 341 | if err != nil { |
| 342 | return err |
| 343 | } |
| 344 | for _, entry := range entries { |
| 345 | if err := ctx.Err(); err != nil { |
| 346 | return err |
| 347 | } |
| 348 | pid, err := strconv.Atoi(entry.Name()) |
| 349 | if err != nil || pid == os.Getpid() { |
| 350 | continue |
| 351 | } |
| 352 | path := "/proc/" + entry.Name() |
| 353 | var st unix.Stat_t |
| 354 | if err := unix.Stat(path, &st); errors.Is(err, unix.ENOENT) { |
| 355 | continue |
| 356 | } else if err != nil { |
| 357 | return err |
| 358 | } |
| 359 | if st.Uid != 0 { |
| 360 | continue |
| 361 | } |
| 362 | raw, err := os.ReadFile(path + "/cmdline") |
| 363 | if errors.Is(err, os.ErrNotExist) { |
| 364 | continue |
| 365 | } |
| 366 | if err != nil { |
| 367 | return err |
| 368 | } |
| 369 | args := strings.Split(string(raw), "\x00") |
| 370 | if len(args) > 1 && (args[1] == "adopt" || args[1] == "system" && len(args) > 2 && (args[2] == "apply" || args[2] == "guard-arm")) { |
| 371 | return errRootMigrationBusy |
| 372 | } |
| 373 | } |
| 374 | return nil |
| 375 | } |
| 376 | |
| 377 | var rootMigrationOfflineUnits = []string{"carlos-adopt.service", "carlos-stage.service", "carlos-config-sync.service", "carlos-restart.service", "carlos-system-update.service", "carlos-system-recover.service"} |
| 378 | |
| 379 | func verifyOfflineRootMigration(ctx context.Context) error { |
| 380 | for _, unit := range rootMigrationOfflineUnits { |
| 381 | raw, err := claimSystemctl(ctx, "show", unit, "--property=LoadState,ActiveState", "--no-pager") |
| 382 | if err != nil { |
| 383 | return err |
| 384 | } |
| 385 | props := string(raw) |
| 386 | if !strings.Contains(props, "LoadState=masked\n") || !(strings.Contains(props, "ActiveState=inactive\n") || strings.Contains(props, "ActiveState=failed\n")) { |
| 387 | return errRootMigrationBusy |
| 388 | } |
| 389 | } |
| 390 | return quiesceLegacyRootMutators(ctx) |
| 391 | } |
| 392 | |
amadan