| 1 | package provision |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "log" |
| 7 | "os" |
| 8 | |
| 9 | "github.com/carlosframework/platform/internal/registry" |
| 10 | "github.com/carlosframework/platform/internal/release" |
| 11 | "github.com/carlosframework/platform/internal/seeds" |
| 12 | ) |
| 13 | |
| 14 | // SeedHooks is what the provision pass needs to act on a seeded record |
| 15 | // (seeded-instances spec §3.2). Nil on ReconcileConfig: no seeded record is |
| 16 | // minted or acted on, which is the safe answer for a box that cannot place |
| 17 | // one. |
| 18 | type SeedHooks struct { |
| 19 | // Data is this region's data bucket (seeds/ lives there). |
| 20 | Data release.Store |
| 21 | // LegacyOwners reads the operator assignment of legacy route names |
| 22 | // (spec §6.4): the swap's only authority over a row filed under the |
| 23 | // operator account. Nil, or an error, defers such a swap. |
| 24 | LegacyOwners func(ctx context.Context) (*seeds.LegacyOwners, error) |
| 25 | // NestedOK is the box's live nested probe, asked after the layout |
| 26 | // intent says nested (spec §5.4 "Layout"). An error defers the swap to |
| 27 | // the next pass; it never refuses (only the intent does). Nil: the |
| 28 | // intent alone decides. |
| 29 | NestedOK func() error |
| 30 | // EnvBefore reports what the host's own env file holds, at the nested |
| 31 | // path beside dbPath, as one of registry's EnvBefore values. The swap |
| 32 | // asks it before the import's row exists and keeps the answer in the |
| 33 | // journal, so a rollback can tell the instance key the import's first |
| 34 | // launch minted from one that was there before (RetireKey). Nil, or an |
| 35 | // error, defers the swap. |
| 36 | EnvBefore func(dbPath, host string) (string, error) |
| 37 | |
| 38 | // The rollback's box effects (spec §5.6, seedrollback.go). All seven |
| 39 | // or none: a box without them leaves every abandoned import journaled, |
| 40 | // gated and untouched. |
| 41 | // |
| 42 | // FenceForRollback fences and stops host under the rollback's fence |
| 43 | // generation, by the layout pass's rule: a fence already holding gen is |
| 44 | // this rollback's own from an earlier pass. proceed false defers the |
| 45 | // rollback, and the hook has lifted any fence it installed. |
| 46 | FenceForRollback func(ctx context.Context, host string, gen uint64) (proceed bool, err error) |
| 47 | // LiftFence lifts the fence gen, if it is the one host holds. |
| 48 | LiftFence func(host string, gen uint64) |
| 49 | // RemoveLocal removes the import's local data for its row (the |
| 50 | // database, -wal, -shm, <db>.seed, litestream's metadata and any seed |
| 51 | // fetch leftovers) through instancefs. Nothing there is not an error. |
| 52 | RemoveLocal func(rt registry.Route) error |
| 53 | // DeleteReplica deletes host's instances/ replica in the region data |
| 54 | // bucket. Nothing there is not an error. |
| 55 | DeleteReplica func(ctx context.Context, host string) error |
| 56 | // NextFenceGen mints a fence generation no fence on this box holds, |
| 57 | // never one it minted before, across agent restarts: the journal |
| 58 | // stores it, and a resumed rollback recognises and lifts its fence by |
| 59 | // it. An error, or 0, closes nothing this pass. |
| 60 | NextFenceGen func() (uint64, error) |
| 61 | // LocalSeedID is the seed id the <db>.seed marker beside dbPath names |
| 62 | // ("" for none or no marker), read through instancefs. An error is a |
| 63 | // marker that could not be read. |
| 64 | LocalSeedID func(dbPath string) (string, error) |
| 65 | // RetireKey retires the instance key the import created in the host's |
| 66 | // own env file beside dbPath, by the journal's EnvBefore: the whole |
| 67 | // file when there was none, its key lines when the file had no key, |
| 68 | // nothing when the key was there before or the journal does not say. |
| 69 | // It never removes a key or a line that predates the import, and it is |
| 70 | // not part of RemoveLocal. Nothing there is not an error. |
| 71 | RetireKey func(dbPath, host, envBefore string) error |
| 72 | } |
| 73 | |
| 74 | type seedVerdict int |
| 75 | |
| 76 | const ( |
| 77 | seedOrdinary seedVerdict = iota |
| 78 | seedSkip |
| 79 | seedFinishAbort |
| 80 | seedSwap |
| 81 | ) |
| 82 | |
| 83 | // seedAction is §3.2's dispatch, run before any other per-record handling. |
| 84 | // row is this box's row for the host, nil when it holds none. nested is |
| 85 | // asked only for a mint, through the pass's lazy layout reader. journalOK |
| 86 | // is whether this pass read the import journal: when it could not, no |
| 87 | // import is acted on, since the journal decides every import it holds a |
| 88 | // row for (spec §5.7) and the no-journal paths must never run while a |
| 89 | // journal row might exist. |
| 90 | func seedAction(rec Record, row *registry.Route, cfg ReconcileConfig, nested func() (bool, error), journalOK bool) seedVerdict { |
| 91 | src := rec.Seed |
| 92 | if src == nil { |
| 93 | return seedOrdinary |
| 94 | } |
| 95 | if cfg.Seeds == nil { |
| 96 | return seedSkip |
| 97 | } |
| 98 | if src.Import() && !journalOK { |
| 99 | log.Printf("provision: %s: import %s left alone this pass: the import journal could not be read", rec.Host, src.ID) |
| 100 | return seedSkip |
| 101 | } |
| 102 | hasRow := row != nil |
| 103 | switch src.State { |
| 104 | case seeds.StateAborted: |
| 105 | return seedFinishAbort |
| 106 | case seeds.StateRolledBack, seeds.StateReserving, seeds.StateAwaitingUpload, seeds.StateUploaded, seeds.StateRefused: |
| 107 | return seedSkip |
| 108 | case seeds.StateReady: |
| 109 | if src.Import() { |
| 110 | return seedSwap // seedswap.go |
| 111 | } |
| 112 | if hasRow { |
| 113 | return seedOrdinary |
| 114 | } |
| 115 | if cfg.ClaimEnabled { |
| 116 | log.Printf("provision: %s: not minting a seeded instance: claim control is on for this box", rec.Host) |
| 117 | return seedSkip |
| 118 | } |
| 119 | // Only an activator-managed row is minted nested (routeFor): any |
| 120 | // other row would start without its seed and without the seed |
| 121 | // command, which is an empty database. |
| 122 | if !rec.DerivesPaths() || !rec.HibernateOrDefault() { |
| 123 | log.Printf("provision: %s: not minting a seeded instance: its record does not describe a hibernating instance", rec.Host) |
| 124 | return seedSkip |
| 125 | } |
| 126 | if ok, err := nested(); err != nil || !ok { |
| 127 | log.Printf("provision: %s: not minting a seeded instance: this box does not mint nested (%v)", rec.Host, err) |
| 128 | return seedSkip |
| 129 | } |
| 130 | return seedOrdinary |
| 131 | case seeds.StateSwapped, seeds.StatePlaced: |
| 132 | // Before committed, an import's row is made only by the swap, |
| 133 | // gated (spec §5.4, §8). A box with no row, or the swapping box |
| 134 | // after its gated row was removed, must not mint one: routeFor |
| 135 | // never sets the gate, so ordinary traffic would reach an |
| 136 | // unverified import. |
| 137 | if src.Import() && !hasRow { |
| 138 | log.Printf("provision: %s: not minting import %s: it is %s but not committed, and only its swap makes its row", rec.Host, src.ID, src.State) |
| 139 | return seedSkip |
| 140 | } |
| 141 | // The swap's row is gated until commit. An ungated row here was |
| 142 | // put there by hand, and adopting or converging it would serve |
| 143 | // the unverified import to ordinary traffic. |
| 144 | if src.Import() && row.ImportGate == "" { |
| 145 | log.Printf("provision: %s: not converging import %s: it is %s but not committed, and the row on this box has no import gate", rec.Host, src.ID, src.State) |
| 146 | return seedSkip |
| 147 | } |
| 148 | return seedOrdinary |
| 149 | case seeds.StateCommitted: |
| 150 | return seedOrdinary |
| 151 | } |
| 152 | log.Printf("provision: %s: seed %s is in an unknown state %q; leaving it alone", rec.Host, src.ID, src.State) |
| 153 | return seedSkip |
| 154 | } |
| 155 | |
| 156 | // retireAbortedSeedRow removes the row of an aborted --from seed that this |
| 157 | // box minted (task-4 review I1). The seed can be aborted after its publish |
| 158 | // landed (a lost publish answer), and then a box may have minted the row and |
| 159 | // be running it, holding the host's lease and writing its replica. The |
| 160 | // console's finish deletes that replica, so the row must stop first. The |
| 161 | // record stays aborted this pass; a later pass that finds no row rolls the |
| 162 | // seed back (finishAbortedSeed). |
| 163 | // |
| 164 | // It removes the row exactly as the removal pass removes a row whose record |
| 165 | // is gone, with the same authority: a Provisioned row of this pass's own |
| 166 | // kind and scope, not a running operation's participant, through |
| 167 | // claimRetire (reg.Delete, which the activator follows by stopping it). It |
| 168 | // also requires the row to be this record's (account and app). A row it |
| 169 | // may not remove is left alone, logged, and the seed stays aborted. |
| 170 | // |
| 171 | // Only --from seeds: an import's row is the instance the import replaces, |
| 172 | // which its rollback restores (Plan 7) and never removes. |
| 173 | func retireAbortedSeedRow(ctx context.Context, reg *registry.Registry, cfg ReconcileConfig, rec Record, rt registry.Route) { |
| 174 | if rt.Kind == SidecarKind || !rt.Provisioned || !cfg.inScope(rt.Region, rt.Pool, everywhereRoute(rt)) || |
| 175 | rt.Account != rec.Account || rt.App != rec.App { |
| 176 | log.Printf("provision: %s: aborted seed %s has a row this pass may not remove; leaving the seed aborted", rec.Host, rec.Seed.ID) |
| 177 | return |
| 178 | } |
| 179 | if cfg.KeepParticipant != nil && cfg.KeepParticipant(rt.Host) { |
| 180 | log.Printf("provision: %s: aborted seed %s: not removing its row: participant of a running operation", rec.Host, rec.Seed.ID) |
| 181 | return |
| 182 | } |
| 183 | if cfg.DryRun { |
| 184 | log.Printf("provision: dry-run: would remove %s route %s (its seed %s was aborted)", rt.Kind, rt.Host, rec.Seed.ID) |
| 185 | return |
| 186 | } |
| 187 | if err := cfg.claimRetire(ctx, reg, rt); err != nil { |
| 188 | log.Printf("provision: remove %s route %s (its seed %s was aborted): %v", rt.Kind, rt.Host, rec.Seed.ID, err) |
| 189 | return |
| 190 | } |
| 191 | log.Printf("provision: removed %s route %s (its seed %s was aborted); the next pass rolls it back", rt.Kind, rt.Host, rec.Seed.ID) |
| 192 | } |
| 193 | |
| 194 | // finishAbortedSeed is §3.2's no-journal path for an aborted seed: remove |
| 195 | // a --from seed's local database (purgeAbortedFromLocal), delete this |
| 196 | // seed's snapshot (only this seed's id, codex R4-1), then CAS the record to |
| 197 | // rolled-back. Idempotent: an absent snapshot and a lost CAS are both done. |
| 198 | // The console finishes from rolled-back (Plan 8). |
| 199 | // |
| 200 | // A host or id that fails seeds.Check names no snapshot (no valid key |
| 201 | // could hold one), so nothing is deleted and the record is still rolled |
| 202 | // back. |
| 203 | func finishAbortedSeed(ctx context.Context, s release.Store, reg *registry.Registry, cfg ReconcileConfig, rec Record) { |
| 204 | if cfg.DryRun { |
| 205 | log.Printf("provision: dry-run: would finish aborted seed %s of %s", rec.Seed.ID, rec.Host) |
| 206 | return |
| 207 | } |
| 208 | if rec.Seed.Kind == seeds.KindReplica && !purgeAbortedFromLocal(ctx, reg, cfg, rec) { |
| 209 | return |
| 210 | } |
| 211 | if err := seeds.Check(rec.Host, rec.Seed.ID); err != nil { |
| 212 | log.Printf("provision: %s: aborted seed %q names no snapshot: %v", rec.Host, rec.Seed.ID, err) |
| 213 | } else { |
| 214 | key := seeds.SnapshotKey(rec.Host, rec.Seed.ID) |
| 215 | if err := cfg.Seeds.Data.Delete(ctx, key); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 216 | log.Printf("provision: %s: delete aborted seed's snapshot: %v", rec.Host, err) |
| 217 | return |
| 218 | } |
| 219 | } |
| 220 | if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedRolledBack(rec.Seed.ID)); err != nil && !errors.Is(err, ErrSeedConflict) { |
| 221 | log.Printf("provision: %s: mark seed %s rolled back: %v", rec.Host, rec.Seed.ID, err) |
| 222 | return |
| 223 | } |
| 224 | log.Printf("provision: %s: aborted seed %s rolled back (no box had swapped it)", rec.Host, rec.Seed.ID) |
| 225 | } |
| 226 | |
| 227 | // rollbackArmed reports whether h has the rollback's seven box hooks. |
| 228 | func rollbackArmed(h *SeedHooks) bool { |
| 229 | return h != nil && h.FenceForRollback != nil && h.LiftFence != nil && h.RemoveLocal != nil && h.DeleteReplica != nil && |
| 230 | h.NextFenceGen != nil && h.LocalSeedID != nil && h.RetireKey != nil |
| 231 | } |
| 232 | |
| 233 | // purgeAbortedFromLocal removes an aborted --from seed's local database |
| 234 | // before the seed is rolled back (final review F2), on a pass that found |
| 235 | // no row for the host. The seed may have been aborted after its publish |
| 236 | // landed (a lost publish answer), and a wake then placed its database at |
| 237 | // the host's default nested path, where a later plain instance at the |
| 238 | // host would open it as a warm cache. The console's finish deletes the |
| 239 | // replica and the record, so nothing else would ever remove the file. |
| 240 | // |
| 241 | // It is Plan 7's rule for an import whose row is gone (removeOrphanedLocal), |
| 242 | // decided from the file: the database goes only when its <db>.seed marker |
| 243 | // names this seed and the host still has no row, with the host fenced and |
| 244 | // its stop proven. The marker is on disk, so this holds after a restart |
| 245 | // too. A marker that is absent or names another seed does not prove the |
| 246 | // data is this seed's: logged, and the seed rolls back with the file left. |
| 247 | // A marker that cannot be read, a fence that does not proceed, a row that |
| 248 | // appears or a removal that fails defers the rollback (ok false). |
| 249 | // |
| 250 | // A box without the rollback's hooks places no seed (they are armed with |
| 251 | // the seed reader, and only claim control leaves them off, which refuses |
| 252 | // every seed), so it has nothing of this seed's to remove: ok. |
| 253 | func purgeAbortedFromLocal(ctx context.Context, reg *registry.Registry, cfg ReconcileConfig, rec Record) (ok bool) { |
| 254 | h := cfg.Seeds |
| 255 | if !rollbackArmed(h) { |
| 256 | log.Printf("provision: %s: aborted seed %s: this box cannot read seed markers, and places no seed; nothing local is removed", rec.Host, rec.Seed.ID) |
| 257 | return true |
| 258 | } |
| 259 | db := NestedDBPath(cfg.DataDir, rec.Host) |
| 260 | ours := func() (bool, bool) { |
| 261 | id, err := h.LocalSeedID(db) |
| 262 | if err != nil { |
| 263 | log.Printf("provision: %s: aborted seed %s stays aborted: the seed marker beside %s cannot be read: %v", rec.Host, rec.Seed.ID, db, err) |
| 264 | return false, false |
| 265 | } |
| 266 | if id != rec.Seed.ID { |
| 267 | log.Printf("provision: %s: aborted seed %s rolls back with the database at %s left: its seed marker names %q, not this seed", rec.Host, rec.Seed.ID, db, id) |
| 268 | return false, true |
| 269 | } |
| 270 | return true, true |
| 271 | } |
| 272 | if mine, ok := ours(); !mine { |
| 273 | return ok |
| 274 | } |
| 275 | gen, err := h.NextFenceGen() |
| 276 | if err == nil && gen == 0 { |
| 277 | err = errors.New("the generation source answered 0") |
| 278 | } |
| 279 | if err != nil { |
| 280 | log.Printf("provision: %s: aborted seed %s stays aborted: no fence generation: %v", rec.Host, rec.Seed.ID, err) |
| 281 | return false |
| 282 | } |
| 283 | proceed, err := h.FenceForRollback(ctx, rec.Host, gen) |
| 284 | defer h.LiftFence(rec.Host, gen) // a hook that failed may have fenced first |
| 285 | if err != nil || !proceed { |
| 286 | log.Printf("provision: %s: aborted seed %s stays aborted: the host is not fenced and stopped yet (%v)", rec.Host, rec.Seed.ID, err) |
| 287 | return false |
| 288 | } |
| 289 | // Under the fence: still no row, and the marker still names this seed. |
| 290 | row, err := reg.Get(rec.Host) |
| 291 | if err != nil { |
| 292 | log.Printf("provision: %s: aborted seed %s stays aborted: the route could not be read: %v", rec.Host, rec.Seed.ID, err) |
| 293 | return false |
| 294 | } |
| 295 | if row != nil { |
| 296 | log.Printf("provision: %s: aborted seed %s stays aborted: a route appeared at the host, so its local database is left", rec.Host, rec.Seed.ID) |
| 297 | return false |
| 298 | } |
| 299 | if mine, ok := ours(); !mine { |
| 300 | return ok |
| 301 | } |
| 302 | if err := h.RemoveLocal(registry.Route{Host: rec.Host, Kind: "instance", DBPath: db}); err != nil { |
| 303 | log.Printf("provision: %s: aborted seed %s stays aborted: remove its local data: %v", rec.Host, rec.Seed.ID, err) |
| 304 | return false |
| 305 | } |
| 306 | log.Printf("provision: %s: aborted seed %s has no route on this box: its local data is removed", rec.Host, rec.Seed.ID) |
| 307 | return true |
| 308 | } |
| 309 | |
| 310 | // journalPass is spec §5.7: the box's import journal is read before the |
| 311 | // records are acted on, every pass, and each row drives its host (§5.4 |
| 312 | // "Crashes", §5.6). handled hosts are skipped by the record loop; |
| 313 | // journaled hosts (every journal row) by the removal phase, since the |
| 314 | // journal owns the row until the row is gone. |
| 315 | // |
| 316 | // The journal is read whatever cfg.Seeds is (preflight M1): with seed |
| 317 | // hooks unarmed every journaled host is still protected and handled, |
| 318 | // with no action. With claim control on for this box it is the same: |
| 319 | // every journaled host is protected and handled, and nothing moves |
| 320 | // forward (no rollback, no redone record CAS, no cleared gate). ok is |
| 321 | // false when the journal cannot be read. Reconcile then treats the pass |
| 322 | // as a failed read for removal, and seedAction acts on no import. |
| 323 | // |
| 324 | // A journal row at swapped is decided by the record that answers to it, |
| 325 | // which is only ever the record under the journal's own key |
| 326 | // (journalRecord): another record for the same host can only vouch for |
| 327 | // the import, never revoke it. |
| 328 | // |
| 329 | // - gone, no seed, another seed, or this seed aborted, refused or rolled |
| 330 | // back: the import is abandoned, so it is rolled back. A refused |
| 331 | // record with a journal row is a refusal that lost the race to this |
| 332 | // box's swap: the journal wins; |
| 333 | // - this seed at ready: the swap's record CAS is redone; |
| 334 | // - this seed at committed: the gate is cleared and the journal row |
| 335 | // dropped (CommitImport), and the loop converges the row; |
| 336 | // - this seed at swapped or placed: the loop handles it as usual. |
| 337 | // |
| 338 | // The loop converges a host only when the pass's own record for it is the |
| 339 | // journal's: when another record won the host, or the journal's record is |
| 340 | // out of scope, the host stays handled, so no other record touches the |
| 341 | // row. Every other case leaves the host alone this pass, logged. |
| 342 | func journalPass(ctx context.Context, s release.Store, reg *registry.Registry, cfg ReconcileConfig, |
| 343 | recs map[string]Record, readFailed bool, protect map[string]bool) (handled, journaled map[string]bool, ok bool) { |
| 344 | handled, journaled = map[string]bool{}, map[string]bool{} |
| 345 | rows, err := reg.ImportJournals() |
| 346 | if err != nil { |
| 347 | log.Printf("provision: the import journal could not be read: %v (no import is acted on and no route is removed this pass)", err) |
| 348 | return handled, journaled, false |
| 349 | } |
| 350 | for _, j := range rows { |
| 351 | journaled[j.Host] = true |
| 352 | handled[j.Host] = true // cleared below for the two cases the loop converges |
| 353 | if cfg.Seeds == nil { |
| 354 | log.Printf("provision: %s: import %s is journaled, but this box cannot act on imports; its row is left alone", j.Host, j.SeedID) |
| 355 | continue |
| 356 | } |
| 357 | // With claim control on, a journal row is only protected (merge |
| 358 | // review S2). The box arms no swap and no rollback there, and the |
| 359 | // claim console refuses commit, abort and finish, so nothing this |
| 360 | // pass could do would be one step of a whole: redoing the swap's |
| 361 | // record CAS writes a record, and clearing a committed gate opens |
| 362 | // the row to ordinary traffic, each on a box whose claim has not |
| 363 | // taken the host. The host stays handled (the loop converges |
| 364 | // nothing) and journaled (the removal phase leaves its row), until |
| 365 | // the claim's own seed algorithm exists (#250). Activation refuses |
| 366 | // while a journal row exists, so this is an import that was caught |
| 367 | // by a claim that came on anyway. |
| 368 | if cfg.ClaimEnabled { |
| 369 | log.Printf("provision: %s: import %s is journaled at %s and claim control is on for this box; its row is protected and nothing else is done", j.Host, j.SeedID, j.State) |
| 370 | continue |
| 371 | } |
| 372 | rec, inPass, known := journalRecord(ctx, s, j, recs, readFailed, protect) |
| 373 | if !known { |
| 374 | log.Printf("provision: %s: import %s left alone this pass: its record could not be read", j.Host, j.SeedID) |
| 375 | continue |
| 376 | } |
| 377 | // journalRecord answers only with the journal's own record; this |
| 378 | // holds before any decision below, rollbacks first of all. |
| 379 | if rec != nil && RecordKey(rec.Account, rec.App, rec.Host) != j.RecordKey { |
| 380 | log.Printf("provision: %s: import %s left alone: the journal holds it under %s, not %s", j.Host, j.SeedID, |
| 381 | j.RecordKey, RecordKey(rec.Account, rec.App, rec.Host)) |
| 382 | continue |
| 383 | } |
| 384 | switch j.State { |
| 385 | case registry.JournalCleanupPending, registry.JournalRollingBack: |
| 386 | startRollback(ctx, s, reg, cfg, j, rec) |
| 387 | continue |
| 388 | case registry.JournalSwapped: |
| 389 | default: |
| 390 | log.Printf("provision: %s: import %s is journaled in an unknown state %q; leaving it alone", j.Host, j.SeedID, j.State) |
| 391 | continue |
| 392 | } |
| 393 | var seed *SeedSource |
| 394 | if rec != nil { |
| 395 | seed = rec.Seed |
| 396 | } |
| 397 | switch { |
| 398 | case seed == nil || seed.ID != j.SeedID: |
| 399 | startRollback(ctx, s, reg, cfg, j, rec) |
| 400 | case seed.State == seeds.StateAborted || seed.State == seeds.StateRefused || seed.State == seeds.StateRolledBack: |
| 401 | startRollback(ctx, s, reg, cfg, j, rec) |
| 402 | case seed.State == seeds.StateReady: |
| 403 | if cfg.DryRun { |
| 404 | log.Printf("provision: dry-run: would redo %s's swap record CAS for import %s", j.Host, j.SeedID) |
| 405 | } else { |
| 406 | markSwapped(ctx, s, *rec) |
| 407 | } |
| 408 | case seed.State == seeds.StateCommitted: |
| 409 | if cfg.DryRun { |
| 410 | log.Printf("provision: dry-run: would clear %s's import gate", j.Host) |
| 411 | } else if err := reg.CommitImport(j.Host, j.SeedID); err != nil { |
| 412 | log.Printf("provision: %s: clear the import gate: %v", j.Host, err) |
| 413 | } else { |
| 414 | log.Printf("provision: %s: import %s committed: its gate is cleared and its journal row dropped", j.Host, j.SeedID) |
| 415 | } |
| 416 | handled[j.Host] = !inPass |
| 417 | case seed.State == seeds.StateSwapped || seed.State == seeds.StatePlaced: |
| 418 | handled[j.Host] = !inPass |
| 419 | default: |
| 420 | // reserving, awaiting-upload and uploaded come before any swap. |
| 421 | log.Printf("provision: %s: import %s is journaled but its record says %s; leaving it alone", j.Host, j.SeedID, seed.State) |
| 422 | } |
| 423 | } |
| 424 | return handled, journaled, true |
| 425 | } |
| 426 | |
| 427 | // startRollback hands an abandoned import to its rollback, which a dry |
| 428 | // run never starts. |
| 429 | func startRollback(ctx context.Context, s release.Store, reg *registry.Registry, cfg ReconcileConfig, j registry.ImportJournal, rec *Record) { |
| 430 | if cfg.DryRun { |
| 431 | log.Printf("provision: dry-run: would roll back %s's import %s", j.Host, j.SeedID) |
| 432 | return |
| 433 | } |
| 434 | rollbackImport(ctx, s, reg, cfg, j, rec) |
| 435 | } |
| 436 | |
| 437 | // journalRecord is the record under the journal's own key, nil when it is |
| 438 | // gone, and known false when this pass cannot tell. inPass says it is the |
| 439 | // record the pass holds for the host, so the loop may converge with it. |
| 440 | // |
| 441 | // recs holds the first in-scope record per host that read cleanly, which |
| 442 | // need not be the journal's: another app's or account's record can win a |
| 443 | // duplicate host, or hold the host while the journal's record is out of |
| 444 | // this box's scope. So recs answers only when its record's key is the |
| 445 | // journal's; otherwise the record is read by the journal's key, and only |
| 446 | // os.ErrNotExist means gone. A protected host, a failed read, a journal |
| 447 | // row with no key, and a body under the key that names another host or |
| 448 | // key all decide nothing this pass: a committed import must never be |
| 449 | // rolled back on a guess. |
| 450 | func journalRecord(ctx context.Context, s release.Store, j registry.ImportJournal, recs map[string]Record, |
| 451 | readFailed bool, protect map[string]bool) (rec *Record, inPass, known bool) { |
| 452 | if protect[j.Host] || j.RecordKey == "" { |
| 453 | return nil, false, false |
| 454 | } |
| 455 | if r, ok := recs[j.Host]; ok && RecordKey(r.Account, r.App, r.Host) == j.RecordKey { |
| 456 | return &r, true, true |
| 457 | } |
| 458 | if readFailed { |
| 459 | return nil, false, false |
| 460 | } |
| 461 | r, err := journalRecordByKey(ctx, s, j) |
| 462 | if err != nil { |
| 463 | log.Printf("provision: %s: the import's record %s is not one this pass can use (%v)", j.Host, j.RecordKey, err) |
| 464 | return nil, false, false |
| 465 | } |
| 466 | return r, false, true |
| 467 | } |
| 468 | |
amadan