| 1 | package instancectl |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "net" |
| 8 | "time" |
| 9 | ) |
| 10 | |
| 11 | // Legacy run start attempts (#292, Astra round 2 P1, the owner's ruling). |
| 12 | // |
| 13 | // The agent proves an aborted start's unit stopped through /stop before it |
| 14 | // gives the lease back. Cancelling its /start does not join this side's |
| 15 | // handler, which could still submit the systemd start after that /stop had |
| 16 | // answered, if it had not yet seen the disconnect; systemd's replacement of |
| 17 | // a queued start by a stop only covers jobs that already exist. So, for the |
| 18 | // original run unit (carlos-instance@, generation 0) only, start and stop |
| 19 | // are ordered per host here: |
| 20 | // |
| 21 | // - every /start registers an attempt with a context /stop can cancel, |
| 22 | // and checks that context immediately before it submits the start; |
| 23 | // - a /stop fences the host: new starts are refused while it runs, every |
| 24 | // attempt in flight is cancelled and joined (bounded), and only then is |
| 25 | // the unit stopped, which replaces any job a joined attempt submitted; |
| 26 | // - a start whose request was accepted at or before the moment a stop |
| 27 | // fences the host (the accepted-connection watermark, connSeq) is an |
| 28 | // earlier request, and is refused, however late its handler registers; |
| 29 | // one accepted after it runs normally. Serve answers one request per |
| 30 | // connection, so a connection's accept order is its request's. |
| 31 | // |
| 32 | // Once /stop has answered, no start from an earlier request is in flight or |
| 33 | // can be submitted afterwards. Generation units, seeds, sidecars and the |
| 34 | // claim verbs are untouched. |
| 35 | |
| 36 | // legacyJoinBudget bounds how long a /stop waits for the start attempts it |
| 37 | // cancelled to return. One that is still running then may yet submit, so |
| 38 | // the stop answers an error and the agent keeps the lease. |
| 39 | const legacyJoinBudget = 10 * time.Second |
| 40 | |
| 41 | // legacyAttempt is one in-flight legacy run start. |
| 42 | type legacyAttempt struct { |
| 43 | cancel context.CancelFunc |
| 44 | done chan struct{} |
| 45 | } |
| 46 | |
| 47 | // connSeqKey carries a connection's accept order in its request contexts. |
| 48 | type connSeqKey struct{} |
| 49 | |
| 50 | // nextConnSeq hands out the next accept-order number (from 1). |
| 51 | func (s *Server) nextConnSeq() uint64 { return s.connSeq.Add(1) } |
| 52 | |
| 53 | // connContext is Serve's ConnContext: the claim verbs' connection value, |
| 54 | // and the connection's accept order. ConnContext runs on the accept loop, |
| 55 | // once per connection, in accept order. |
| 56 | func (s *Server) connContext(ctx context.Context, c net.Conn) context.Context { |
| 57 | return context.WithValue(requestConnContext(ctx, c), connSeqKey{}, s.nextConnSeq()) |
| 58 | } |
| 59 | |
| 60 | // requestSeq is a request's accept order; 0 when its connection was not |
| 61 | // stamped (a handler called outside Serve), which no fence refuses. |
| 62 | func requestSeq(ctx context.Context) uint64 { |
| 63 | n, _ := ctx.Value(connSeqKey{}).(uint64) |
| 64 | return n |
| 65 | } |
| 66 | |
| 67 | // errLegacyFenced refuses a start while a stop of its host runs, or one |
| 68 | // whose request came before a stop of its host. |
| 69 | var errLegacyFenced = errors.New("a stop of this host's run unit has run since this start was asked for") |
| 70 | |
| 71 | // beginLegacyStart registers a legacy run start of host for a request of |
| 72 | // accept order seq. Its context is parent's, cancellable by a /stop of the |
| 73 | // host; finish must be called when the handler returns. |
| 74 | func (s *Server) beginLegacyStart(parent context.Context, host string, seq uint64) (context.Context, func(), error) { |
| 75 | s.legacyMu.Lock() |
| 76 | defer s.legacyMu.Unlock() |
| 77 | if s.legacyStopping[host] > 0 || (seq != 0 && seq <= s.legacyBarrier[host]) { |
| 78 | return nil, nil, errLegacyFenced |
| 79 | } |
| 80 | ctx, cancel := context.WithCancel(parent) |
| 81 | a := &legacyAttempt{cancel: cancel, done: make(chan struct{})} |
| 82 | if s.legacyStarts == nil { |
| 83 | s.legacyStarts = make(map[string]map[*legacyAttempt]struct{}) |
| 84 | } |
| 85 | if s.legacyStarts[host] == nil { |
| 86 | s.legacyStarts[host] = make(map[*legacyAttempt]struct{}) |
| 87 | } |
| 88 | s.legacyStarts[host][a] = struct{}{} |
| 89 | finish := func() { |
| 90 | s.legacyMu.Lock() |
| 91 | delete(s.legacyStarts[host], a) |
| 92 | if len(s.legacyStarts[host]) == 0 { |
| 93 | delete(s.legacyStarts, host) |
| 94 | } |
| 95 | s.legacyMu.Unlock() |
| 96 | cancel() |
| 97 | close(a.done) |
| 98 | } |
| 99 | return ctx, finish, nil |
| 100 | } |
| 101 | |
| 102 | // fenceLegacyStarts is a /stop's first step for host: refuse new starts |
| 103 | // while the stop runs, refuse every start accepted at or before this moment |
| 104 | // (the accepted-connection watermark, read under legacyMu, so no start can |
| 105 | // register between the watermark and the snapshot), cancel the attempts in |
| 106 | // flight and join them, bounded by the join budget and ctx. The fence is |
| 107 | // per host and in memory only: release lifts the refusal of new starts when |
| 108 | // the stop is done, and only connections accepted before the watermark |
| 109 | // stay refused. An error means an attempt did not return in time and may |
| 110 | // still submit. |
| 111 | func (s *Server) fenceLegacyStarts(ctx context.Context, host string) (func(), error) { |
| 112 | s.legacyMu.Lock() |
| 113 | seq := s.connSeq.Load() |
| 114 | if s.legacyStopping == nil { |
| 115 | s.legacyStopping = make(map[string]int) |
| 116 | } |
| 117 | if s.legacyBarrier == nil { |
| 118 | s.legacyBarrier = make(map[string]uint64) |
| 119 | } |
| 120 | s.legacyStopping[host]++ |
| 121 | if seq > s.legacyBarrier[host] { |
| 122 | s.legacyBarrier[host] = seq |
| 123 | } |
| 124 | var inFlight []*legacyAttempt |
| 125 | for a := range s.legacyStarts[host] { |
| 126 | inFlight = append(inFlight, a) |
| 127 | } |
| 128 | s.legacyMu.Unlock() |
| 129 | release := func() { |
| 130 | s.legacyMu.Lock() |
| 131 | if s.legacyStopping[host]--; s.legacyStopping[host] <= 0 { |
| 132 | delete(s.legacyStopping, host) |
| 133 | } |
| 134 | s.legacyMu.Unlock() |
| 135 | } |
| 136 | budget := legacyJoinBudget |
| 137 | if s.legacyJoin > 0 { |
| 138 | budget = s.legacyJoin |
| 139 | } |
| 140 | timer := time.NewTimer(budget) |
| 141 | defer timer.Stop() |
| 142 | for _, a := range inFlight { |
| 143 | a.cancel() |
| 144 | select { |
| 145 | case <-a.done: |
| 146 | case <-timer.C: |
| 147 | return release, fmt.Errorf("a start of %s is still in flight after %s", host, budget) |
| 148 | case <-ctx.Done(): |
| 149 | return release, fmt.Errorf("a start of %s is still in flight: %w", host, ctx.Err()) |
| 150 | } |
| 151 | } |
| 152 | return release, nil |
| 153 | } |
| 154 | |
amadan