| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "os" |
| 9 | "path/filepath" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/hostfloor" |
| 15 | ) |
| 16 | |
| 17 | // Executable updates and claim acquisition share root's transaction lock. |
| 18 | // The root marker persists through the update journal's terminal state. |
| 19 | // Agent files may add a refusal, but never establish an idle census. |
| 20 | |
| 21 | const ( |
| 22 | updateBegunFile = "update-begun.json" |
| 23 | hostexecDir = ".hostexec" |
| 24 | ) |
| 25 | |
| 26 | // systemGuardDir is /opt/carlos/system/guard, beside root's update state. |
| 27 | func systemGuardDir(optDir string) string { return filepath.Join(optDir, "system", "guard") } |
| 28 | |
| 29 | func updateBegunPath(optDir string) string { |
| 30 | return filepath.Join(systemGuardDir(optDir), updateBegunFile) |
| 31 | } |
| 32 | |
| 33 | type updateBegun struct { |
| 34 | Target string `json:"target"` |
| 35 | PromotedAt time.Time `json:"promoted_at"` |
| 36 | At time.Time `json:"at"` |
| 37 | } |
| 38 | |
| 39 | // beginExecutableChange is §5.10's root side, run under the apply lock |
| 40 | // just before the first box-changing step: write the marker durably, then |
| 41 | // look for participation. On a refusal the marker is removed again and the |
| 42 | // error, wrapping errSystemDeferred, says why. The caller records it as |
| 43 | // "deferred", never "refused": a refusal is terminal for that promote, and |
| 44 | // this one must be retried once the operation ends (spec §5.10: the |
| 45 | // update is refused and retried later). The agent's leg re-requests a |
| 46 | // deferred apply on its backoff ladder, at most every five minutes. |
| 47 | func (a *systemApplier) beginExecutableChange(ctx context.Context, target string, promotedAt time.Time) error { |
| 48 | required, err := a.participationDisk().ParticipationRequired(ctx) |
| 49 | if err != nil { |
| 50 | return err |
| 51 | } |
| 52 | if !required && !a.regionHA { |
| 53 | return nil |
| 54 | } |
| 55 | active, err := a.activeFloor() |
| 56 | if err != nil { |
| 57 | return err |
| 58 | } |
| 59 | if active { |
| 60 | // beginGuard already persisted the reservation ID. Retain it until |
| 61 | // authenticated completion, including when participation appears late. |
| 62 | tx, err := a.guardTransaction() |
| 63 | if err != nil { |
| 64 | return err |
| 65 | } |
| 66 | raw, err := os.ReadFile(filepath.Join(a.floor.Dir, updateBegunFile)) |
| 67 | if err != nil { |
| 68 | return err |
| 69 | } |
| 70 | var begun struct { |
| 71 | ID string `json:"id"` |
| 72 | } |
| 73 | if json.Unmarshal(raw, &begun) != nil || begun.ID != tx.ID { |
| 74 | return hostfloor.ErrFloor |
| 75 | } |
| 76 | if err := a.controllersIdle(ctx); err != nil { |
| 77 | return fmt.Errorf("%w: %v", errSystemDeferred, err) |
| 78 | } |
| 79 | return nil |
| 80 | } |
| 81 | if !a.regionHA { |
| 82 | return a.participationDisk().WithParticipation(ctx, func(ctx context.Context, l rootparticipation.Ledger) error { |
| 83 | return l.BeginUpdate(ctx, "system-"+target) |
| 84 | }, func() error { return nil }) |
| 85 | } |
| 86 | writeMarker := func() error { |
| 87 | if err := os.MkdirAll(systemGuardDir(a.optDir), 0o755); err != nil { |
| 88 | return err |
| 89 | } |
| 90 | return writeJSONDurable(updateBegunPath(a.optDir), updateBegun{Target: target, PromotedAt: promotedAt, At: a.ops.now().UTC()}, 0o644) |
| 91 | } |
| 92 | err = a.participationDisk().BeginParticipation(ctx, "system-"+target, a.dataDir, writeMarker, func() error { |
| 93 | if err := writeMarker(); err != nil { |
| 94 | return err |
| 95 | } |
| 96 | if reason := a.participationDisk().CheckParticipation(ctx, a.dataDir, false); reason != nil { |
| 97 | if err := a.clearLegacyUpdateBegun(); err != nil { |
| 98 | a.ops.logf("system apply: %v", err) |
| 99 | } |
| 100 | return reason |
| 101 | } |
| 102 | return nil |
| 103 | }) |
| 104 | if err != nil { |
| 105 | return fmt.Errorf("%w: %v", errSystemDeferred, err) |
| 106 | } |
| 107 | return nil |
| 108 | } |
| 109 | |
| 110 | // clearUpdateBegun removes both markers only after positive root settlement. |
| 111 | func (a *systemApplier) clearUpdateBegun(ctx context.Context) error { |
| 112 | return a.participationDisk().WithParticipation(ctx, func(ctx context.Context, l rootparticipation.Ledger) error { |
| 113 | id, err := l.UpdateID() |
| 114 | if errors.Is(err, os.ErrNotExist) { |
| 115 | if _, e := os.Lstat(updateBegunPath(a.optDir)); errors.Is(e, os.ErrNotExist) { |
| 116 | return nil |
| 117 | } |
| 118 | id = "recover-legacy" |
| 119 | err = l.BeginUpdate(ctx, id) |
| 120 | } |
| 121 | if err != nil { |
| 122 | return err |
| 123 | } |
| 124 | return l.EndUpdateWith(ctx, id, func() error { |
| 125 | if _, e := os.Lstat(updateBegunPath(a.optDir)); errors.Is(e, os.ErrNotExist) { |
| 126 | return nil |
| 127 | } |
| 128 | return hostfloor.RemoveDurable(updateBegunPath(a.optDir)) |
| 129 | }) |
| 130 | }, a.clearLegacyUpdateBegun) |
| 131 | } |
| 132 | |
| 133 | func (a *systemApplier) clearLegacyUpdateBegun() error { |
| 134 | err := os.Remove(updateBegunPath(a.optDir)) |
| 135 | if errors.Is(err, os.ErrNotExist) { |
| 136 | return nil |
| 137 | } |
| 138 | if err != nil { |
| 139 | return fmt.Errorf("remove %s: %w", updateBegunPath(a.optDir), err) |
| 140 | } |
| 141 | return syncDir(systemGuardDir(a.optDir)) |
| 142 | } |
| 143 | |
| 144 | // settleUpdateBegun clears the marker once no update is in progress: no |
| 145 | // journal means the update finished, rolled back, was recovered, or never |
| 146 | // got past its first step. A kept journal (a rollback that could not |
| 147 | // finish) keeps the marker with it. Run with or without CARLOS_REGION_HA, |
| 148 | // so a marker left behind after the flag is turned off still goes. |
| 149 | // |
| 150 | // On a census box the settlement goes through settleWithoutJournal (Astra |
| 151 | // round 3): a legacy marker with no update record and no journal has |
| 152 | // clearUpdateBegun begin root's update record, and a census refusal there |
| 153 | // withdrew it and left acquisitions open while the error was only logged. |
| 154 | // Now a census refusal comes back as errSettlementDeferred with the |
| 155 | // "recover-legacy" barrier held, and a barrier that could not be written as |
| 156 | // errAcquisitionBarrier; the caller decides what each means for the run. |
| 157 | // Every other failure is logged and swallowed, as on main. |
| 158 | func (a *systemApplier) settleUpdateBegun(ctx context.Context) error { |
| 159 | if active, err := a.activeFloor(); err != nil || active { |
| 160 | return nil |
| 161 | } |
| 162 | if j, err := readSystemJournal(a.optDir); err != nil || j != nil { |
| 163 | return nil |
| 164 | } |
| 165 | err := a.settleWithoutJournal(ctx, "recover-legacy", a.clearUpdateBegun(ctx)) |
| 166 | if err == nil { |
| 167 | return nil |
| 168 | } |
| 169 | if errors.Is(err, errSettlementDeferred) || errors.Is(err, errAcquisitionBarrier) { |
| 170 | return err |
| 171 | } |
| 172 | a.ops.logf("system apply: %v", err) |
| 173 | return nil |
| 174 | } |
| 175 | |
| 176 | // closeGuardRollback is closeGuardJournal for the backward paths: rollback |
| 177 | // and backward recovery. On a floor-armed box it first records in root's |
| 178 | // marker that the transaction rolled back, before the journal goes, so the |
| 179 | // completion that follows is recorded for an image that is not the target |
| 180 | // (Opus re-review M1-4). A marker for another transaction refuses and keeps |
| 181 | // the journal. With no armed floor this is closeGuardJournal, as on main. |
| 182 | func (a *systemApplier) closeGuardRollback(ctx context.Context) error { |
| 183 | active, err := a.activeFloor() |
| 184 | if err != nil { |
| 185 | return err |
| 186 | } |
| 187 | if active { |
| 188 | tx, err := a.guardTransaction() |
| 189 | if err != nil { |
| 190 | return err |
| 191 | } |
| 192 | // No marker means none was begun, and settleGuard then records no |
| 193 | // completion: the reservation stays held, which is the safe side. |
| 194 | if err := hostfloor.MarkRolledBack(a.floor.Dir, tx.ID); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 195 | return err |
| 196 | } |
| 197 | } |
| 198 | return a.closeGuardJournal(ctx) |
| 199 | } |
| 200 | |
| 201 | func (a *systemApplier) participationDisk() hostfloor.Disk { |
| 202 | if a.floor != nil { |
| 203 | return *a.floor |
| 204 | } |
| 205 | return hostfloor.Disk{Participation: a.participation} |
| 206 | } |
| 207 | |
| 208 | func (a *systemApplier) checkUnarmedParticipationRecovery(ctx context.Context) error { |
| 209 | required, err := a.participationDisk().ParticipationRequired(ctx) |
| 210 | if err != nil || !required { |
| 211 | return err |
| 212 | } |
| 213 | j, err := readSystemJournal(a.optDir) |
| 214 | if err != nil { |
| 215 | return err |
| 216 | } |
| 217 | recover, reserve := false, false |
| 218 | err = a.participationDisk().WithParticipation(ctx, func(ctx context.Context, l rootparticipation.Ledger) error { |
| 219 | c, err := l.RecoveryCensus() |
| 220 | if err != nil { |
| 221 | return err |
| 222 | } |
| 223 | if c.Pending == "acquisition" { |
| 224 | recover = true |
| 225 | } |
| 226 | _, err = l.UpdateID() |
| 227 | if errors.Is(err, os.ErrNotExist) { |
| 228 | if j == nil { |
| 229 | return nil |
| 230 | } |
| 231 | reserve = true |
| 232 | } else if err != nil { |
| 233 | return err |
| 234 | } |
| 235 | recover = true |
| 236 | return nil |
| 237 | }, func() error { return nil }) |
| 238 | if err != nil { |
| 239 | return err |
| 240 | } |
| 241 | if recover { |
| 242 | if err := a.controllersRecovered(ctx); err != nil { |
| 243 | if j == nil { |
| 244 | err = a.settleWithoutJournal(ctx, "recover-system", err) |
| 245 | } |
| 246 | return err |
| 247 | } |
| 248 | } |
| 249 | if reserve { |
| 250 | return a.participationDisk().WithParticipation(ctx, func(ctx context.Context, l rootparticipation.Ledger) error { |
| 251 | return l.BeginUpdate(ctx, "recover-system") |
| 252 | }, func() error { return nil }) |
| 253 | } |
| 254 | return nil |
| 255 | } |
| 256 | |
amadan