| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "crypto/sha256" |
| 7 | "encoding/hex" |
| 8 | "encoding/json" |
| 9 | "errors" |
| 10 | "fmt" |
| 11 | "io" |
| 12 | "os" |
| 13 | "path/filepath" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/hostclaim" |
| 17 | "github.com/carlosframework/platform/internal/hostguard" |
| 18 | "github.com/carlosframework/platform/internal/provision" |
| 19 | "github.com/carlosframework/platform/internal/registry" |
| 20 | "github.com/carlosframework/platform/internal/release" |
| 21 | "github.com/carlosframework/platform/internal/rollout" |
| 22 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 23 | "github.com/carlosframework/platform/internal/rootstate" |
| 24 | "github.com/carlosframework/platform/internal/safefile" |
| 25 | ) |
| 26 | |
| 27 | type rootUpdateRecord struct { |
| 28 | RetryMirror *rootLegacyRetry `json:",omitempty"` |
| 29 | Schema int |
| 30 | Installation string |
| 31 | Route registry.Route |
| 32 | Build release.BuildStateCopy |
| 33 | Consumed *release.Consumed |
| 34 | Served map[string]rootServedBuild |
| 35 | } |
| 36 | |
| 37 | type rootServedBuild struct { |
| 38 | Digest, Invocation, Source string |
| 39 | Epoch uint64 |
| 40 | } |
| 41 | |
| 42 | type rootUpdateSource struct { |
| 43 | enforceFailures bool |
| 44 | heldGeneration *release.GenerationBinding |
| 45 | |
| 46 | legacy []byte |
| 47 | ready func(registry.Route) error |
| 48 | publish func(string, rootRetryStatusRecord) error |
| 49 | source rootRestartRequestSource |
| 50 | binding *proofBindingSource |
| 51 | state rootstate.Store |
| 52 | book *rollout.RootFailures |
| 53 | store release.Store |
| 54 | pub ed25519.PublicKey |
| 55 | observe func(registry.Route) (liveBuild, error) |
| 56 | data string |
| 57 | } |
| 58 | |
| 59 | func rootUpdatesMigration(c rootparticipation.Census) string { |
| 60 | return fmt.Sprintf("%s/%s/%d", c.Box, c.Enrollment, c.Migration) |
| 61 | } |
| 62 | |
| 63 | func rootFailureLegacy(data string) ([]byte, error) { |
| 64 | f, err := safefile.OpenFile(filepath.Join(data, rollout.FailureFile), os.O_RDONLY, 0) |
| 65 | if errors.Is(err, os.ErrNotExist) { |
| 66 | return nil, nil |
| 67 | } |
| 68 | if err != nil { |
| 69 | return nil, err |
| 70 | } |
| 71 | defer f.Close() |
| 72 | b, err := io.ReadAll(io.LimitReader(f, rootstate.MaxRecord+1)) |
| 73 | if len(b) > rootstate.MaxRecord { |
| 74 | return nil, os.ErrPermission |
| 75 | } |
| 76 | return b, err |
| 77 | } |
| 78 | |
| 79 | func productionRootUpdates(dd string, probes ...func() (rootparticipation.Ledger, bool, error)) release.RootUpdateOpener { |
| 80 | return productionRootUpdatesHeld(dd, nil, probes...) |
| 81 | } |
| 82 | |
| 83 | func productionRootUpdatesHeld(dd string, held *release.GenerationBinding, probes ...func() (rootparticipation.Ledger, bool, error)) release.RootUpdateOpener { |
| 84 | return func(ctx context.Context, store release.Store, pub ed25519.PublicKey) (*release.RootUpdateInputs, error) { |
| 85 | probe := rootparticipation.OpenForUpdate |
| 86 | if len(probes) > 1 || len(probes) == 1 && probes[0] == nil { |
| 87 | return nil, os.ErrPermission |
| 88 | } |
| 89 | if len(probes) == 1 { |
| 90 | probe = probes[0] |
| 91 | } |
| 92 | ledger, required, err := probe() |
| 93 | if ledger.State != nil { |
| 94 | defer ledger.State.Close() |
| 95 | } |
| 96 | if err != nil { |
| 97 | return nil, err |
| 98 | } |
| 99 | if !required { |
| 100 | return nil, nil |
| 101 | } |
| 102 | c, err := ledger.Census() |
| 103 | if err != nil { |
| 104 | return nil, err |
| 105 | } |
| 106 | if !c.Complete || c.NeverEnrolled { |
| 107 | return nil, os.ErrPermission |
| 108 | } |
| 109 | state, err := rootstate.Open(rollout.RootFailuresDir, false, false) |
| 110 | if err != nil { |
| 111 | return nil, err |
| 112 | } |
| 113 | fail := func(err error) (*release.RootUpdateInputs, error) { state.Close(); return nil, err } |
| 114 | book, err := rollout.OpenRootFailures(state, rootUpdatesMigration(c)) |
| 115 | if err != nil { |
| 116 | return fail(err) |
| 117 | } |
| 118 | legacy, _ := rootFailureLegacy(dd) |
| 119 | fences, err := rootstate.Open("/var/lib/carlos-claims/authority/fences", false, false) |
| 120 | if err != nil { |
| 121 | return fail(err) |
| 122 | } |
| 123 | binding := productionProofBindingSource(store, systemdUnits{}, release.OptDir()) |
| 124 | binding.cacheIndex = false |
| 125 | binding.claim = func(ctx context.Context, host string) (hostclaim.Observed, error) { |
| 126 | return hostclaim.Read(ctx, store, host) |
| 127 | } |
| 128 | binding.box = func(context.Context) (string, error) { return c.Box, nil } |
| 129 | observer := productionProofObserver(release.OptDir()) |
| 130 | observer.resolve = binding.resolve |
| 131 | p := rootUpdateSource{legacy: legacy, ready: func(rt registry.Route) error { return rootUpdateReady(ctx, rt) }, publish: publishRootRetry, source: rootRestartRequestSource{state: fences, store: store, census: func() (rootparticipation.Census, error) { return c, nil }}, binding: binding, state: state, book: book, store: store, pub: pub, data: dd, observe: func(rt registry.Route) (liveBuild, error) { return observer.observe(rt, 0) }} |
| 132 | p.heldGeneration = held |
| 133 | return &release.RootUpdateInputs{Resolve: p.resolve, Close: func() error { return errors.Join(fences.Close(), state.Close()) }}, nil |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | func (p rootUpdateSource) resolve(ctx context.Context, host string) (release.RootUpdateInput, error) { |
| 138 | var out release.RootUpdateInput |
| 139 | var in hostguard.RootInstallation |
| 140 | var err error |
| 141 | if p.heldGeneration != nil { |
| 142 | in, err = p.source.launchInstallation(ctx, host, *p.heldGeneration) |
| 143 | } else { |
| 144 | in, err = p.source.installation(ctx, host) |
| 145 | } |
| 146 | if err != nil { |
| 147 | return out, err |
| 148 | } |
| 149 | if in.Empty || in.Record != provision.RecordKey(in.Account, in.App, host) { |
| 150 | return out, os.ErrPermission |
| 151 | } |
| 152 | rec, etag, err := provision.GetRecordETag(ctx, p.store, in.Account, in.App, host) |
| 153 | if err != nil { |
| 154 | return out, err |
| 155 | } |
| 156 | inc, err := hostclaim.Incarnation(rec.Account, rec.App, rec.Host, rec.CreatedAt) |
| 157 | if err != nil || inc != in.Incarnation || etag == "" { |
| 158 | return out, os.ErrPermission |
| 159 | } |
| 160 | if _, err = p.binding.claimIdentity(ctx, rec); err != nil { |
| 161 | return out, err |
| 162 | } |
| 163 | var policy *provision.UpdatePolicy |
| 164 | manifest, err := provision.GetManifest(ctx, p.store, rec.Account, rec.App) |
| 165 | if err == nil { |
| 166 | policy = manifest.Updates |
| 167 | } else if !errors.Is(err, os.ErrNotExist) { |
| 168 | return out, err |
| 169 | } |
| 170 | if err = provision.ValidatePolicy(policy); err != nil { |
| 171 | return out, err |
| 172 | } |
| 173 | rt := provision.RouteFor(*rec, p.data, "", true, policy) |
| 174 | // A remote proposal cannot supply protected local lifecycle or served facts. |
| 175 | rt.GoodVersion, rt.GoodDigest, rt.GoodDemand, rt.ServingVersion, rt.BuildState = "", "", "", "", "" |
| 176 | installation := rootHistoryInstallation(in) |
| 177 | key := rootSettingsKey(host, installation) |
| 178 | var saved rootUpdateRecord |
| 179 | raw, err := p.state.Read(key) |
| 180 | if errors.Is(err, os.ErrNotExist) { |
| 181 | old, readErr := p.state.Read("settings-" + host + ".json") |
| 182 | if readErr == nil { |
| 183 | var previous rootUpdateRecord |
| 184 | if rootparticipation.Decode(old, &previous) != nil || previous.Schema != 1 { |
| 185 | return out, os.ErrPermission |
| 186 | } |
| 187 | if previous.Installation == installation { |
| 188 | raw, err = old, nil |
| 189 | } |
| 190 | } else if !errors.Is(readErr, os.ErrNotExist) { |
| 191 | return out, readErr |
| 192 | } |
| 193 | } |
| 194 | if err == nil { |
| 195 | if rootparticipation.Decode(raw, &saved) != nil || saved.Schema != 1 || saved.Installation != installation || saved.Route.Host != host || saved.Route.Account != rec.Account || saved.Route.App != rec.App { |
| 196 | return out, os.ErrPermission |
| 197 | } |
| 198 | } else if errors.Is(err, os.ErrNotExist) { |
| 199 | saved = rootUpdateRecord{Schema: 1, Installation: installation, Route: rt, Served: map[string]rootServedBuild{}} |
| 200 | } else { |
| 201 | return out, err |
| 202 | } |
| 203 | save := func() error { |
| 204 | body, err := json.Marshal(saved) |
| 205 | if err != nil { |
| 206 | return err |
| 207 | } |
| 208 | return p.state.Write(key, body) |
| 209 | } |
| 210 | // Observe canonical process, executable digest and readiness together. Registry |
| 211 | // good-build fields and remote build-state never enter this record. |
| 212 | if rt.Unit != "" && p.observe != nil { |
| 213 | live, e := p.observe(rt) |
| 214 | if e == nil && live.binding.authorizedRecovery() && live.binding.RecordKey == in.Record && live.binding.RecordETag == etag && live.binding.Digest == live.SHA256 && live.binding.Version == live.Version && live.binding.Route.Host == host && live.binding.Route.Account == rec.Account && live.binding.Route.App == rec.App && live.Version != "" && live.SHA256 != "" && live.invocation != "" && live.binding.Source != "" { |
| 215 | if saved.Served == nil { |
| 216 | saved.Served = map[string]rootServedBuild{} |
| 217 | } |
| 218 | epoch := p.book.RestrictionEpoch(rec.Account, rec.App, live.Version, host) |
| 219 | saved.Served[live.Version] = rootServedBuild{live.SHA256, live.invocation, live.binding.Source, epoch} |
| 220 | saved.Build = release.BuildStateCopy{Known: true, State: &release.BuildState{Schema: 1, Host: host, LastExecuted: live.Version}} |
| 221 | if saved.Consumed != nil && saved.Consumed.Resolved != nil && saved.Consumed.Resolved.Target == live.Version { |
| 222 | saved.Build.State.Resolved = saved.Consumed.Resolved |
| 223 | saved.Build.State.Completed = saved.Consumed.Resolved.ID |
| 224 | } |
| 225 | } |
| 226 | } |
| 227 | saved.Route = rt |
| 228 | if err = save(); err != nil { |
| 229 | return out, err |
| 230 | } |
| 231 | retries, _, err := release.LoadHostRetries(ctx, p.store, rec.Account, rec.App) |
| 232 | if err != nil { |
| 233 | return out, err |
| 234 | } |
| 235 | for _, r := range retries.Requests { |
| 236 | if r.Scope.Kind == release.RestartScopeRetry && r.Scope.Value == host { |
| 237 | if grant, e := release.VerifyHostRetry(p.pub, r.Scope.Signed, rec.Account, rec.App, host, time.Now()); e == nil { |
| 238 | exact := p.book.ID() == grant.Book && p.book.PublishedEpoch(rec.Account, rec.App, grant.Build, host) == grant.Epoch |
| 239 | if exact { |
| 240 | legacyBook, epoch := rollout.LegacyRetryIdentity(p.legacy, rec.Account, rec.App) |
| 241 | if legacyBook != "" && epoch != 0 { |
| 242 | saved.RetryMirror = &rootLegacyRetry{Build: grant.Build, Book: legacyBook, Epoch: epoch} |
| 243 | } |
| 244 | if err = save(); err != nil { |
| 245 | return out, err |
| 246 | } |
| 247 | } |
| 248 | if err = p.book.VerifiedRetry(p.pub, r.Scope.Signed, rec.Account, rec.App, host, time.Now()); err != nil { |
| 249 | return out, err |
| 250 | } |
| 251 | } |
| 252 | } |
| 253 | } |
| 254 | ptr, e := release.Channel(ctx, p.store, p.pub, rec.Account, rec.App, rec.Channel) |
| 255 | if e == nil { |
| 256 | if served, ok := saved.Served[ptr.Version]; ok && served.Digest != "" { |
| 257 | if m, e := release.AuthorizedManifest(ctx, p.store, p.pub, rec.Account, rec.App, ptr.Version); e == nil && m.Artifacts[release.AdoptPlatform()].SHA256 == served.Digest { |
| 258 | if _, err = p.book.ClearHoldRanOnPointer(host, rec.Account, rec.App, ptr.PromotedAt); err != nil { |
| 259 | return out, err |
| 260 | } |
| 261 | } |
| 262 | } |
| 263 | if _, err = p.book.ClearOnPointer(rec.Account, rec.App, ptr.Version, ptr.PromotedAt); err != nil { |
| 264 | return out, err |
| 265 | } |
| 266 | } |
| 267 | out = release.RootUpdateInput{Route: rt, Build: saved.Build, Consumed: saved.Consumed, Failed: func(build string) bool { |
| 268 | if !p.enforceFailures { |
| 269 | return false |
| 270 | } |
| 271 | served, ok := saved.Served[build] |
| 272 | ok = ok && served.Digest != "" && served.Invocation != "" && served.Source != "" && served.Epoch == p.book.RestrictionEpoch(rec.Account, rec.App, build, host) |
| 273 | if ok { |
| 274 | manifest, err := release.AuthorizedManifest(ctx, p.store, p.pub, rec.Account, rec.App, build) |
| 275 | ok = err == nil && manifest.Artifacts[release.AdoptPlatform()].SHA256 == served.Digest |
| 276 | } |
| 277 | if mark, held := p.book.HoldRanOn(host); held && mark.Account == rec.Account && mark.App == rec.App && mark.Build != build && saved.Build.State != nil && saved.Build.State.LastExecuted == build { |
| 278 | return true |
| 279 | } |
| 280 | return p.book.SaveOwed() || p.book.Blocks(rec.Account, rec.App, build, host, ok) || p.book.HeldDown(rec.Account, rec.App, build, host) |
| 281 | }, Save: func(c release.Consumed) error { saved.Consumed = &c; return save() }} |
| 282 | out.Begin = func(version, digest, attempt string) (func(string, error, func() error) error, error) { |
| 283 | return p.beginRuntime(&saved, save, version, digest, attempt) |
| 284 | } |
| 285 | projectedVersion := rec.Pin |
| 286 | if projectedVersion == "" && ptr != nil { |
| 287 | projectedVersion = ptr.Version |
| 288 | } |
| 289 | if err = p.publishRetry(saved, projectedVersion); err != nil && p.enforceFailures { |
| 290 | return release.RootUpdateInput{}, err |
| 291 | } |
| 292 | if saved.Build.State != nil { |
| 293 | out.Installed = saved.Build.State.LastExecuted |
| 294 | } |
| 295 | return out, nil |
| 296 | } |
| 297 | |
| 298 | func initializeMigrationFailures(dd string, execution rootstate.Store) error { |
| 299 | c, err := (rootparticipation.Ledger{State: execution}).Census() |
| 300 | if err != nil { |
| 301 | return err |
| 302 | } |
| 303 | if !c.Complete || c.NeverEnrolled { |
| 304 | return os.ErrPermission |
| 305 | } |
| 306 | s, err := rootstate.Open(rollout.RootFailuresDir, true, false) |
| 307 | if err != nil { |
| 308 | return err |
| 309 | } |
| 310 | defer s.Close() |
| 311 | raw, err := rootFailureLegacy(dd) |
| 312 | if err != nil { |
| 313 | return err |
| 314 | } |
| 315 | _, err = rollout.InitializeRootFailures(s, rootUpdatesMigration(c), raw) |
| 316 | return err |
| 317 | } |
| 318 | |
| 319 | func rootSettingsKey(host, installation string) string { |
| 320 | sum := sha256.Sum256([]byte(host + "\x00" + installation)) |
| 321 | return "settings-" + hex.EncodeToString(sum[:]) + ".json" |
| 322 | } |
| 323 | |
amadan