| 1 | package instancectl |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "fmt" |
| 6 | "net/http" |
| 7 | "strings" |
| 8 | "sync" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/provision" |
| 12 | ) |
| 13 | |
| 14 | func (s *Server) cmdPoll() time.Duration { |
| 15 | if s.cmdPollEvery > 0 { |
| 16 | return s.cmdPollEvery |
| 17 | } |
| 18 | return 250 * time.Millisecond |
| 19 | } |
| 20 | |
| 21 | func (s *Server) cmdWait() time.Duration { |
| 22 | if s.cmdWaitBound > 0 { |
| 23 | return s.cmdWaitBound |
| 24 | } |
| 25 | return 30 * time.Second |
| 26 | } |
| 27 | |
| 28 | func (s *Server) cmdStartWait() time.Duration { |
| 29 | if s.cmdStartBound > 0 { |
| 30 | return s.cmdStartBound |
| 31 | } |
| 32 | return 10 * time.Second |
| 33 | } |
| 34 | |
| 35 | // handleCmd starts host's cmd unit and answers at once with the new |
| 36 | // invocation (run spec §5.4); /seed blocks, this never does. The agent |
| 37 | // stops a cmd unit after every run, so a unit still loaded (active, |
| 38 | // activating, deactivating) belongs to a run this request must not end: it |
| 39 | // is refused, never restarted. A failed one is reset first, or start would |
| 40 | // keep its old result. |
| 41 | func (s *Server) handleCmd(w http.ResponseWriter, r *http.Request) { |
| 42 | req, err := parseRequest(r, verbShape{tier: true, exe: true, cmd: true}) |
| 43 | if err != nil { |
| 44 | http.Error(w, err.Error(), http.StatusBadRequest) |
| 45 | return |
| 46 | } |
| 47 | // One start per host at a time: a second request waits here, then reads |
| 48 | // the unit loaded and is refused, instead of racing the first one's |
| 49 | // invocation check and stopping its command. |
| 50 | lock, _ := s.cmdLocks.LoadOrStore(req.host, new(sync.Mutex)) |
| 51 | lock.(*sync.Mutex).Lock() |
| 52 | defer lock.(*sync.Mutex).Unlock() |
| 53 | ctx := r.Context() |
| 54 | unit := UnitName(RoleCmd, req.host) |
| 55 | switch state := s.showValue(ctx, "ActiveState", unit); state { |
| 56 | case "active", "activating", "deactivating", "reloading", "refreshing": |
| 57 | http.Error(w, "cmd: "+unit+" is "+state, http.StatusConflict) |
| 58 | return |
| 59 | case "failed": |
| 60 | if out, err := s.Systemctl(ctx, "reset-failed", unit); err != nil { |
| 61 | http.Error(w, "reset-failed: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway) |
| 62 | return |
| 63 | } |
| 64 | } |
| 65 | // With generations on, the agent names the serving generation's staged |
| 66 | // build, which root validates before it lays out the mount, as for the |
| 67 | // legacy sidecar (zero-downtime spec §7.5). Without one, the live link, |
| 68 | // as the seed. |
| 69 | if s.refuseStagedExe(w, req, unit) { |
| 70 | return |
| 71 | } |
| 72 | // The command's executable is laid out at its own source (merge ruling |
| 73 | // M4), never at the host's app, which an instance or sidecar start may |
| 74 | // be rewriting for a different build at this moment. |
| 75 | if s.refuseStart(w, req.host, unit, func() error { return s.ensureCmdExeAt(req.host, req.exe) }) { |
| 76 | return |
| 77 | } |
| 78 | // A command runs beside an awake instance, whose own start set the |
| 79 | // slice. With generations on the slice is the generation parent, and a |
| 80 | // swap may have raised it to two tiers (zero-downtime spec §7.3); its |
| 81 | // restore is the swap owner's. So a populated slice is left alone, as |
| 82 | // startGen and the sidecar's start leave it, and only a cold one gets |
| 83 | // the tier. |
| 84 | if s.slicePopulated(req.host) { |
| 85 | s.logf("instance-control: %s: slice populated, leaving its limit to the instance's owner", req.host) |
| 86 | } else if err := s.setTier(ctx, req.host, req.tier); err != nil { |
| 87 | http.Error(w, "set-property: "+err.Error(), http.StatusBadGateway) |
| 88 | return |
| 89 | } |
| 90 | before := s.showValue(ctx, "InvocationID", unit) |
| 91 | if out, err := s.Systemctl(ctx, "start", "--no-block", unit); err != nil { |
| 92 | http.Error(w, "start: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway) |
| 93 | return |
| 94 | } |
| 95 | // --no-block returns once the job is queued; the invocation exists |
| 96 | // once the job runs. |
| 97 | deadline := time.Now().Add(s.cmdStartWait()) |
| 98 | for { |
| 99 | if inv := s.showValue(ctx, "InvocationID", unit); inv != "" && inv != before { |
| 100 | s.logf("instance-control: %s: started %s memory.max=%d", req.host, unit, req.tier.MemoryMax) |
| 101 | writeJSON(w, http.StatusOK, StartResponse{InvocationID: inv}) |
| 102 | return |
| 103 | } |
| 104 | if time.Now().After(deadline) || ctx.Err() != nil { |
| 105 | dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) |
| 106 | s.stopUnit(dctx, unit) |
| 107 | cancel() |
| 108 | http.Error(w, "cmd: "+unit+" showed no new invocation", http.StatusBadGateway) |
| 109 | return |
| 110 | } |
| 111 | time.Sleep(s.cmdPoll()) |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | // handleCmdStatus is the bounded long-poll: it answers as soon as the named |
| 116 | // invocation is done or gone, and running at the bound. |
| 117 | func (s *Server) handleCmdStatus(w http.ResponseWriter, r *http.Request) { |
| 118 | host := r.PathValue("host") |
| 119 | if !provision.ValidHostShape(host) { |
| 120 | http.Error(w, fmt.Sprintf("host %q is not a host", host), http.StatusBadRequest) |
| 121 | return |
| 122 | } |
| 123 | want := r.URL.Query().Get("invocation") |
| 124 | if want == "" { |
| 125 | http.Error(w, "invocation is required", http.StatusBadRequest) |
| 126 | return |
| 127 | } |
| 128 | ctx := r.Context() |
| 129 | unit := UnitName(RoleCmd, host) |
| 130 | deadline := time.Now().Add(s.cmdWait()) |
| 131 | for { |
| 132 | out, err := s.Systemctl(ctx, "show", "-p", "InvocationID,ActiveState,SubState,Result,ExecMainStatus,ExecMainCode", unit) |
| 133 | if err != nil { |
| 134 | http.Error(w, "show: "+err.Error(), http.StatusBadGateway) |
| 135 | return |
| 136 | } |
| 137 | st := cmdStatusOf(parseShow(out), want) |
| 138 | if st.State != CmdRunning || !time.Now().Before(deadline) { |
| 139 | writeJSON(w, http.StatusOK, st) |
| 140 | return |
| 141 | } |
| 142 | select { |
| 143 | case <-ctx.Done(): |
| 144 | return |
| 145 | case <-time.After(s.cmdPoll()): |
| 146 | } |
| 147 | } |
| 148 | } |
| 149 | |
| 150 | // cmdStatusOf reads one show for the named invocation. |
| 151 | func cmdStatusOf(m map[string]string, want string) CmdStatus { |
| 152 | st := CmdStatus{InvocationID: m["InvocationID"], Result: m["Result"], Status: m["ExecMainStatus"], Code: m["ExecMainCode"]} |
| 153 | active, sub := m["ActiveState"], m["SubState"] |
| 154 | switch { |
| 155 | case st.InvocationID != want: |
| 156 | st.State = CmdGone |
| 157 | case active == "failed", active == "inactive", active == "active" && sub == "exited": |
| 158 | st.State = CmdDone |
| 159 | default: |
| 160 | st.State = CmdRunning |
| 161 | } |
| 162 | return st |
| 163 | } |
| 164 | |
| 165 | // cleanCmdExe removes host's cmd executable sources after its cmd unit has |
| 166 | // stopped. It holds the host's cmd lock, so a start cannot be laying them |
| 167 | // out meanwhile, and leaves them for a unit that is not inactive or failed |
| 168 | // (a start that has since begun). |
| 169 | func (s *Server) cleanCmdExe(ctx context.Context, host string) { |
| 170 | lock, _ := s.cmdLocks.LoadOrStore(host, new(sync.Mutex)) |
| 171 | lock.(*sync.Mutex).Lock() |
| 172 | defer lock.(*sync.Mutex).Unlock() |
| 173 | switch s.showValue(ctx, "ActiveState", UnitName(RoleCmd, host)) { |
| 174 | case "inactive", "failed": |
| 175 | s.removeCmdExe(provision.InstanceID(host)) |
| 176 | } |
| 177 | } |
| 178 | |
amadan