| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "slices" |
| 8 | |
| 9 | "github.com/carlosframework/platform/internal/hostbox" |
| 10 | "github.com/carlosframework/platform/internal/hostclaim" |
| 11 | execlease "github.com/carlosframework/platform/internal/hostexec" |
| 12 | "github.com/carlosframework/platform/internal/hostguard" |
| 13 | ) |
| 14 | |
| 15 | func (h ClaimRootHTTP) settleExecution(ctx context.Context, r execlease.ExecutionRecord) error { |
| 16 | o, err := hostclaim.Read(ctx, h.Stop.Coordinator.Evidence, r.Request.Host) |
| 17 | if err != nil { |
| 18 | return err |
| 19 | } |
| 20 | if o.Absent || o.Claim == nil || o.Claim.Host != r.Request.Host || o.Claim.Validate() != nil { |
| 21 | return execlease.ErrState |
| 22 | } |
| 23 | c := o.Claim |
| 24 | if !c.Free() && c.Op.Kind != hostclaim.KindServing && slices.Contains(c.Op.Participants, r.Authority.Box) { |
| 25 | return execlease.ErrInactive |
| 26 | } |
| 27 | if !c.Free() && c.Holder.Incarnation != r.Request.Incarnation && !slices.Contains(c.RetiredIncarnations, r.Request.Incarnation) { |
| 28 | return execlease.ErrState |
| 29 | } |
| 30 | return nil |
| 31 | } |
| 32 | |
| 33 | func (h ClaimRootHTTP) cleanupExecution(ctx context.Context, r execlease.ExecutionRecord) error { |
| 34 | q, a := r.Request, r.Authority |
| 35 | guard := h.Stop.Coordinator.Guard |
| 36 | if h.Launches == nil || guard.Root == nil || h.Reconcile.Census == nil || h.Stop.Coordinator.Kernel == nil { |
| 37 | return execlease.ErrState |
| 38 | } |
| 39 | all, err := h.Launches.HostLaunches(ctx, q.Host) |
| 40 | if err != nil { |
| 41 | return err |
| 42 | } |
| 43 | var launches []hostbox.LaunchInventory |
| 44 | captured := hostguard.Captured{Incarnation: q.Incarnation, Stamp: q.Stamp} |
| 45 | if a.ParticipationOnly && a.Installation != "" { |
| 46 | if a.Kind == hostclaim.KindDeleting || a.Kind == hostclaim.KindParking { |
| 47 | view, viewErr := guard.Root.View(ctx, q.Host, captured) |
| 48 | if viewErr != nil { |
| 49 | return viewErr |
| 50 | } |
| 51 | in := view.Installation |
| 52 | if view.Barrier != nil || in.Incarnation != captured.Incarnation || in.Stamp != captured.Stamp || in.Box != a.Box || in.Enrollment != a.Enrollment || in.Account != a.Account || in.App != a.App || in.Migration != a.Migration { |
| 53 | return execlease.ErrLost |
| 54 | } |
| 55 | } else { |
| 56 | captured, err = guard.Capture(q.Host) |
| 57 | if err != nil { |
| 58 | return err |
| 59 | } |
| 60 | } |
| 61 | if captured.Incarnation != q.Incarnation || fmt.Sprintf("%s/%d/%s/%s", captured.Incarnation, captured.Stamp, a.Box, a.Enrollment) != a.Installation { |
| 62 | return execlease.ErrLost |
| 63 | } |
| 64 | } |
| 65 | for _, launch := range all { |
| 66 | target := launch.Target |
| 67 | if target.Incarnation == captured.Incarnation && target.Stamp == captured.Stamp && target.Box == a.Box && target.Enrollment == a.Enrollment { |
| 68 | launches = append(launches, launch) |
| 69 | } |
| 70 | } |
| 71 | if len(launches) == 0 { |
| 72 | if a.Installation != "" { |
| 73 | view, err := guard.Root.View(ctx, q.Host, captured) |
| 74 | if err != nil { |
| 75 | return err |
| 76 | } |
| 77 | in := view.Installation |
| 78 | if view.Barrier != nil || in.Box != a.Box || in.Enrollment != a.Enrollment || in.Account != a.Account || in.App != a.App || in.Migration != a.Migration || fmt.Sprintf("%s/%d/%s/%s", in.Incarnation, in.Stamp, in.Box, in.Enrollment) != a.Installation { |
| 79 | return ErrClaimStopInventory |
| 80 | } |
| 81 | inventory, revision, err := h.Journals.Inventory.Snapshot(ctx, q.Host, captured) |
| 82 | if err != nil { |
| 83 | return err |
| 84 | } |
| 85 | if !inventory.Complete || revision == 0 || revision != in.InventoryRevision { |
| 86 | return ErrClaimStopInventory |
| 87 | } |
| 88 | kernel, ok := h.Stop.Coordinator.Kernel.(stopNoProcessKernel) |
| 89 | if !ok { |
| 90 | return ErrClaimStopUnavailable |
| 91 | } |
| 92 | boot, err := execlease.BootID() |
| 93 | if err != nil { |
| 94 | return err |
| 95 | } |
| 96 | target := hostguard.FenceTarget{Host: q.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Box: a.Box, Enrollment: a.Enrollment, BootID: boot, DirectoryRevision: inventory.DirectoryRevision} |
| 97 | _, err = kernel.VerifyNoProcess(ctx, target, inventory) |
| 98 | return err |
| 99 | } |
| 100 | if !a.ParticipationOnly { |
| 101 | return ErrClaimStopInventory |
| 102 | } |
| 103 | if _, err := guard.Capture(q.Host); !errors.Is(err, hostguard.ErrNoRecord) { |
| 104 | return ErrClaimStopInventory |
| 105 | } |
| 106 | o, err := hostclaim.Read(ctx, h.Stop.Coordinator.Evidence, q.Host) |
| 107 | if err != nil { |
| 108 | return err |
| 109 | } |
| 110 | return h.rootEmptyCensus().CheckLocal(ctx, o, hostbox.RecordSelection{Host: q.Host, Incarnation: q.Incarnation}) |
| 111 | } |
| 112 | view, err := guard.Root.View(ctx, q.Host, captured) |
| 113 | if err != nil { |
| 114 | return err |
| 115 | } |
| 116 | for _, launch := range launches { |
| 117 | var receipt *hostguard.FenceReceipt |
| 118 | for i := range view.Receipts { |
| 119 | if hostbox.SameLaunchOwner(view.Receipts[i].Target, launch.Target) { |
| 120 | receipt = &view.Receipts[i] |
| 121 | break |
| 122 | } |
| 123 | } |
| 124 | if receipt == nil { |
| 125 | pending := view.Pending |
| 126 | if pending == nil || !hostbox.SameLaunchOwner(*pending, launch.Target) { |
| 127 | next, err := h.beginExecutionRecoveryFence(ctx, launch.Target) |
| 128 | if err != nil { |
| 129 | return err |
| 130 | } |
| 131 | pending = &next |
| 132 | } |
| 133 | if _, err := h.Stop.Coordinator.RecoverStopped(ctx, *pending); err != nil { |
| 134 | return err |
| 135 | } |
| 136 | durable, err := guard.FenceReceiptFor(ctx, *pending) |
| 137 | if err != nil { |
| 138 | return err |
| 139 | } |
| 140 | receipt = &durable |
| 141 | } |
| 142 | if err := verifyRetirementReceipt(ctx, *receipt); err != nil { |
| 143 | return err |
| 144 | } |
| 145 | reader, ok := h.Stop.Coordinator.Inventory.(exactStopInventory) |
| 146 | if !ok { |
| 147 | return ErrClaimStopInventory |
| 148 | } |
| 149 | inventory, err := reader.ReadExactStopInventory(ctx, receipt.Target) |
| 150 | if err != nil { |
| 151 | return err |
| 152 | } |
| 153 | launch, err = h.Launches.Read(ctx, receipt.Target) |
| 154 | if err != nil { |
| 155 | return err |
| 156 | } |
| 157 | if retired, err := retiredExecutionLaunch(view, launch); retired || err != nil { |
| 158 | if err != nil { |
| 159 | return err |
| 160 | } |
| 161 | continue |
| 162 | } |
| 163 | ack := DrainClaimAcknowledgement(receipt.Target.FenceID, launch.Target.ControllerID, launch.ProcessStart, launch.ProducerIDs, launch.Completed) |
| 164 | if _, err := h.Stop.Coordinator.Kernel.VerifyStopped(ctx, receipt.Target, inventory, ack); err != nil { |
| 165 | return err |
| 166 | } |
| 167 | } |
| 168 | return nil |
| 169 | } |
| 170 | |
| 171 | func retiredExecutionLaunch(view hostguard.FenceView, launch hostbox.LaunchInventory) (bool, error) { |
| 172 | t := launch.Target |
| 173 | if !slices.Contains(view.Retired, hostguard.Captured{Incarnation: t.Incarnation, Stamp: t.Stamp}) { |
| 174 | return false, nil |
| 175 | } |
| 176 | if view.Host != t.Host || view.Barrier != nil || !launch.WritersClosed || !launch.Completed.Valid() { |
| 177 | return true, ErrClaimStopInventory |
| 178 | } |
| 179 | for _, r := range launch.Reservations { |
| 180 | if !hostbox.SameLaunchOwner(r.Target, t) || (hostbox.IsUnitReservation(r) && r.State != hostbox.UnitSettled) || (!hostbox.IsUnitReservation(r) && r.State != "settled") { |
| 181 | return true, ErrClaimStopInventory |
| 182 | } |
| 183 | } |
| 184 | return true, nil |
| 185 | } |
| 186 | |
| 187 | func (h ClaimRootHTTP) transitionExecution(ctx context.Context, old execlease.ExecutionRecord, q execlease.ExecutionRequest, next execlease.ExecutionAuthority) error { |
| 188 | a := old.Authority |
| 189 | if old.Request.Host != q.Host || old.Request.Incarnation != q.Incarnation || old.Request.Op != q.Op || a.Box != next.Box || a.Enrollment != next.Enrollment || a.Account != next.Account || a.App != next.App || a.Migration != next.Migration { |
| 190 | return execlease.ErrLost |
| 191 | } |
| 192 | if old.Request.Stamp == q.Stamp && old.Request.Session == q.Session { |
| 193 | if a.Installation == next.Installation && a.ParticipationOnly == next.ParticipationOnly { |
| 194 | return nil |
| 195 | } |
| 196 | if a.ParticipationOnly && next.Installation != "" { |
| 197 | return nil |
| 198 | } |
| 199 | return execlease.ErrLost |
| 200 | } |
| 201 | if next.ParticipationOnly && a.Installation == next.Installation && q.Stamp >= old.Request.Stamp && q.Session >= old.Request.Session { |
| 202 | return nil |
| 203 | } |
| 204 | if next.ParticipationOnly || next.Installation == "" || q.Stamp < old.Request.Stamp || q.Session <= old.Request.Session { |
| 205 | return execlease.ErrLost |
| 206 | } |
| 207 | view, err := h.Stop.Coordinator.Guard.Root.View(ctx, q.Host, hostguard.Captured{Incarnation: q.Incarnation, Stamp: q.Stamp}) |
| 208 | if err != nil { |
| 209 | return err |
| 210 | } |
| 211 | if view.Installation.Stamp != q.Stamp || view.Installation.Box != next.Box || view.Installation.Enrollment != next.Enrollment { |
| 212 | return execlease.ErrLost |
| 213 | } |
| 214 | if a.ParticipationOnly && (a.Installation == "" || a.Installation == next.Installation) { |
| 215 | return nil |
| 216 | } |
| 217 | if q.Stamp == old.Request.Stamp { |
| 218 | if a.Installation != next.Installation { |
| 219 | return execlease.ErrLost |
| 220 | } |
| 221 | return h.cleanupExecution(ctx, old) |
| 222 | } |
| 223 | if !slices.Contains(view.Retired, hostguard.Captured{Incarnation: q.Incarnation, Stamp: old.Request.Stamp}) { |
| 224 | return execlease.ErrLost |
| 225 | } |
| 226 | return nil |
| 227 | } |
| 228 | |
| 229 | func (h ClaimRootHTTP) beginExecutionRecoveryFence(ctx context.Context, target hostguard.FenceTarget) (hostguard.FenceTarget, error) { |
| 230 | o, err := h.Stop.Coordinator.Claims.Read(ctx, target.Host) |
| 231 | if err != nil { |
| 232 | return hostguard.FenceTarget{}, err |
| 233 | } |
| 234 | if o.Absent || o.Claim == nil || o.Claim.Holder == nil || h.Stop.Coordinator.NewID == nil { |
| 235 | return hostguard.FenceTarget{}, ErrClaimStopInventory |
| 236 | } |
| 237 | c := o.Claim |
| 238 | target.ClaimETag, target.ClaimGen = o.ETag, c.Gen |
| 239 | target.Op, target.Phase = c.Op.ID, c.Op.Phase |
| 240 | if c.Op.Kind == hostclaim.KindServing { |
| 241 | target.ObligationID = "" |
| 242 | } else if target.ObligationID == "" { |
| 243 | var ok bool |
| 244 | switch c.Op.Kind { |
| 245 | case hostclaim.KindLanding: |
| 246 | target.ObligationID, ok = hostguard.LandingWriterStop(c, target.Box, target.Region) |
| 247 | case hostclaim.KindRelocating: |
| 248 | target.ObligationID, ok = hostguard.RelocationWriterStop(c, target.Box, target.Region) |
| 249 | case hostclaim.KindRestoring, hostclaim.KindReverting: |
| 250 | target.ObligationID, ok = hostguard.RecoveryWriterStop(c, target.Box, target.Region) |
| 251 | } |
| 252 | if !ok { |
| 253 | return hostguard.FenceTarget{}, ErrClaimStopInventory |
| 254 | } |
| 255 | } |
| 256 | if !hostguard.StopAuthorityMatches(c, target) || !stopOperationTuple(target.Op, target.Phase, target.ObligationID) { |
| 257 | return hostguard.FenceTarget{}, ErrClaimStopInventory |
| 258 | } |
| 259 | target.FenceID = h.Stop.Coordinator.NewID() |
| 260 | return h.Stop.Coordinator.Guard.BeginFence(ctx, target) |
| 261 | } |
| 262 | |
amadan