amadan

carlos / platform Public

deploy, canary: watch a gated host through a gate pass (#164)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Human

Prompt

Can we fix this issue? https://amadan.net/carlos/platform/-/discussion/164

From this commit's own message. The prompt ledger has no entries for it — which is what a squashed, rebased or amended commit looks like, and what every commit looks like in a repo without a ledger.

Summary

The watch fetches a pass for each host the instance list marks gated and sends it on every poll, so the version is verified instead of timing out on the login page. A refused pass (old console, 403) drops the host with a line and exits 0; a cleared gate is watched as open; a transient failure keeps the host watched without a pass. A 401 with no X-Carlos-Version is reported as the gate, never as a wake failure.
Agent notes
Known limits: --timeout over 15m outlives the pass; a static --host custom domain with no record gets no pass. Copy approved in review round 1; the transient-failure line is in round 2.
claude fdd098dc0c7ea79ccc88a89931b0d785c59fe100 parent f2e61fb
4 files changed, +340 −5
  • apimode.go +46 −0
  • canary.go +10 −4
  • deploy.go +88 −1
  • deploy_gate_test.go +196 −0

CI passed — run details

Intent

full ledger: gated-deploy-watch-164

diff --git a/apimode.go b/apimode.go
index 1e13c746e..679479ca9 100644
--- a/apimode.go
+++ b/apimode.go
@@ -3186,6 +3186,52 @@ func (s *apiSession) instanceList(ctx context.Context, a apiAcct, app, match, he
return res.Instances, res.Truncated, res.Total, nil
}
+// errGateOpen is instanceGatePass's answer for a host with no gate left on it.
+var errGateOpen = errors.New("no password gate is set on this host")
+
+// gatePassRefused is instanceGatePass's answer when the console will not
+// issue a pass at all, as distinct from failing to.
+type gatePassRefused struct{ err error }
+
+func (e *gatePassRefused) Error() string { return e.err.Error() }
+func (e *gatePassRefused) Unwrap() error { return e.err }
+
+// instanceGatePass asks the console for a short-lived gate cookie for one
+// gated host (handleAPIInstanceGatePass), so the deploy watch can read the
+// version through the password page instead of timing out on it
+// (discussion #164). The answer is the whole Cookie header value,
+// name=value, ready to send.
+//
+// Two answers are typed, because the watch treats them differently from a
+// failure: a 409 is errGateOpen (the gate was cleared after the deploy
+// listed its hosts, so the host needs no pass), and a 404 or 403 is a
+// *gatePassRefused (a console that predates the door, or a credential it
+// will not give one to). Anything else, a 429 or a 502 or a network error,
+// is returned as it came.
+func (s *apiSession) instanceGatePass(ctx context.Context, a apiAcct, app, host string) (string, error) {
+ status, body, err := s.postJSON(ctx, apiPath(a, app, "/instances/"+url.PathEscape(host)+"/gate-pass"), struct{}{})
+ if err != nil {
+ return "", err
+ }
+ switch status {
+ case http.StatusOK:
+ case http.StatusConflict:
+ return "", errGateOpen
+ case http.StatusNotFound, http.StatusForbidden:
+ return "", &gatePassRefused{apiErrorFromBody(status, body)}
+ default:
+ return "", apiErrorFromBody(status, body)
+ }
+ var pass struct {
+ Name string `json:"name"`
+ Value string `json:"value"`
+ }
+ if err := json.Unmarshal(body, &pass); err != nil || pass.Name == "" || pass.Value == "" {
+ return "", fmt.Errorf("gate pass: unreadable response")
+ }
+ return (&http.Cookie{Name: pass.Name, Value: pass.Value}).String(), nil
+}
+
// apiRouteRow is one row of the console's routes answer
// (internal/console/apiroutes.go). Every field is additive on the wire: a
// console that predates one sends nothing and it decodes as the zero
diff --git a/canary.go b/canary.go
index e295f95f5..061c0a48f 100644
--- a/canary.go
+++ b/canary.go
@@ -465,11 +465,17 @@ func runCanaryDeploy(args []string) {
}
fmt.Fprintf(stdout, "%s %s to %s\n", pal.ok("promoted"), *ver, channel)
- targets := []watchTarget{{Host: host, Origin: watchOrigin(host)}}
- if err = watchVersion(stdout, targets, *ver, *timeout, canaryWatchEvery, nil); err != nil {
- log.Fatalf("canary: %v", err)
+ targets := passGatedTargets(stdout, []watchTarget{{Host: host, Origin: watchOrigin(host)}}, gatedHosts(recs), *ver, channel,
+ func(h string) (string, error) { return sess.instanceGatePass(ctx, acct, app2, h) })
+ // Empty only when the host is gated and the console gave no pass, which
+ // passGatedTargets has already said. "canary live at" would be a claim
+ // nothing checked.
+ if len(targets) > 0 {
+ if err = watchVersion(stdout, targets, *ver, *timeout, canaryWatchEvery, nil); err != nil {
+ log.Fatalf("canary: %v", err)
+ }
+ fmt.Fprintf(stdout, "\n%s https://%s\n", pal.ok("canary live at"), host)
}
- fmt.Fprintf(stdout, "\n%s https://%s\n", pal.ok("canary live at"), host)
mutedf(stdout, "promote it onward when you're happy — it costs no rung: carlos promote --app %s %s <channel>\n", app2, *ver)
}
diff --git a/deploy.go b/deploy.go
index 38874a9ce..21a7d8bbb 100644
--- a/deploy.go
+++ b/deploy.go
@@ -362,6 +362,8 @@ func runDeploy(args []string) {
for _, h := range hosts {
targets = append(targets, watchTarget{Host: h, Origin: watchOrigin(h), Probe: probe})
}
+ targets = passGatedTargets(stdout, targets, gatedHosts(recs), *ver, channel,
+ func(host string) (string, error) { return sess.instanceGatePass(ctx, acct, *app, host) })
diag := &watchDiagnostics{
// A timed-out host's wake reason, from the platform log stream —
// scoped tight (one host, errors only, the watch's own recent past)
@@ -1322,9 +1324,72 @@ func filterChannelHosts(recs []apiInstanceRecord, channel, hostFlag string) ([]s
// criterion at all: it goes green while the old binary is still serving. A
// probe path is answered by the app's OWN process, so its body is the only
// honest answer to "which build is running here".
+//
+// Cookie is a gate pass (name=value) for a host behind a password gate, sent
+// on every poll so the edge lets the watch through to the version it is
+// looking for; Gated says the host was gated when the deploy looked, so a
+// 401 from it can be named for what it is. Both are empty for an open host.
type watchTarget struct {
Host, Origin string
Probe string
+ Cookie string
+ Gated bool
+}
+
+// gatedHosts is the set of instance hosts the console says carry a password
+// gate. A console that predates gates says nothing (Gated nil), which reads
+// as open here: such a console has no way to have set one.
+func gatedHosts(recs []apiInstanceRecord) map[string]bool {
+ out := map[string]bool{}
+ for _, r := range recs {
+ if r.Gated != nil && *r.Gated {
+ out[r.Host] = true
+ }
+ }
+ return out
+}
+
+// passGatedTargets fetches a gate pass for each target behind a password
+// gate (discussion #164). Without one, the gate answers every poll with its
+// login page, and the watch used to time out on a deploy that had landed and
+// blame a wake or a route for it.
+//
+// What happens without a pass depends on why. A console that REFUSES one
+// (a *gatePassRefused: too old for the door, or a credential it will not
+// serve) drops the host from the watch with a line saying so: the ship and
+// the promote have landed, and nothing from outside the gate can say more.
+// A host whose gate has since been cleared (errGateOpen) is watched like any
+// open host. Any other failure, a busy or unreachable console, keeps the
+// host in the watch without a pass, so the deploy fails on the gate's 401
+// rather than going green on a check it never made.
+func passGatedTargets(w io.Writer, targets []watchTarget, gated map[string]bool, ver, channel string, pass func(host string) (string, error)) []watchTarget {
+ out := make([]watchTarget, 0, len(targets))
+ for _, t := range targets {
+ if !gated[t.Host] {
+ out = append(out, t)
+ continue
+ }
+ cookie, err := pass(t.Host)
+ var refused *gatePassRefused
+ switch {
+ case errors.Is(err, errGateOpen):
+ out = append(out, t)
+ continue
+ case errors.As(err, &refused):
+ mutedf(w, "%s has a password gate and the console could not give the watch a pass: %v\n %s is on %s. its version can't be checked from outside, so this host is not watched\n",
+ t.Host, err, ver, channel)
+ continue
+ case err != nil:
+ mutedf(w, "%s has a password gate and the console could not give the watch a pass: %v\n watching it anyway. the gate will answer until the pass comes through, so this is likely to time out\n",
+ t.Host, err)
+ out = append(out, t)
+ continue
+ }
+ mutedf(w, "%s has a password gate. the watch checks the version with a 15-minute pass from the console\n", t.Host)
+ t.Cookie, t.Gated = cookie, true
+ out = append(out, t)
+ }
+ return out
}
// watchOrigin maps an instance host to the origin the watch dials. In
@@ -1408,11 +1473,33 @@ func watchVersion(w io.Writer, targets []watchTarget, ver string, timeout, inter
path = t.Probe
}
url := fmt.Sprintf("%s%s?carlos-deploy-probe=%d", t.Origin, path, n)
- resp, err := client.Get(url)
+ req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
results[i] = result{diag: "unreachable: " + err.Error()}
return
}
+ if t.Cookie != "" {
+ req.Header.Set("Cookie", t.Cookie)
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ results[i] = result{diag: "unreachable: " + err.Error()}
+ return
+ }
+ // A 401 with no version header is the password gate's login
+ // page, on either leg: the edge stamps X-Carlos-Version before
+ // it proxies, so an app's own 401 carries the header, and only
+ // a response the edge wrote itself, ahead of the app, lacks it.
+ if resp.StatusCode == http.StatusUnauthorized && resp.Header.Get("X-Carlos-Version") == "" {
+ io.CopyN(io.Discard, resp.Body, 4096)
+ resp.Body.Close()
+ if t.Gated {
+ results[i] = result{diag: "the password gate refused the watch's pass (HTTP 401). the password may have changed during the deploy, or the pass expired"}
+ } else {
+ results[i] = result{diag: "HTTP 401 with no version header. a password gate may be set on this host. run `carlos gate list` to check"}
+ }
+ return
+ }
if t.Probe != "" {
// The app's own process answers here, so the body IS the
// truth — no header/route-row disagreement to reconcile
diff --git a/deploy_gate_test.go b/deploy_gate_test.go
new file mode 100644
index 000000000..8a75e09ae
--- /dev/null
+++ b/deploy_gate_test.go
@@ -0,0 +1,196 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/gate"
+)
+
+// gatedEdge stands in for the edge in front of a gated static route: the
+// real internal/gate decides, and only an admitted request reaches the
+// handler that stamps the version (discussion #164).
+func gatedEdge(t *testing.T, host, hash, ver string) *httptest.Server {
+ t.Helper()
+ g := gate.New()
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if !g.Admit(w, r, host, hash) {
+ return
+ }
+ w.Header().Set("X-Carlos-Version", ver)
+ w.WriteHeader(http.StatusOK)
+ }))
+ t.Cleanup(srv.Close)
+ return srv
+}
+
+func passFor(host, hash string) string {
+ return (&http.Cookie{Name: gate.CookieName, Value: gate.Mint(host, hash, time.Now().Add(time.Minute))}).String()
+}
+
+// TestWatchVersionThroughGateWithPass: with the console's pass on every
+// poll, the gate lets the watch through and the deploy goes live, where it
+// used to time out on the login page.
+func TestWatchVersionThroughGateWithPass(t *testing.T) {
+ const host, hash = "report.bab.test", "$2a$10$stand-in-hash"
+ srv := gatedEdge(t, host, hash, "v2")
+ var out bytes.Buffer
+ err := watchVersion(&out, []watchTarget{{Host: host, Origin: srv.URL, Cookie: passFor(host, hash), Gated: true}},
+ "v2", 2*time.Second, 10*time.Millisecond, nil)
+ if err != nil {
+ t.Fatalf("a watch holding a valid pass must go live: %v", err)
+ }
+ if !strings.Contains(out.String(), host) {
+ t.Fatalf("live line missing: %q", out.String())
+ }
+}
+
+// TestWatchVersionGatedDiagnostics: a gate's 401 is never reported as a wake
+// or route failure. Holding a pass it refuses, the report says the pass was
+// refused; holding none, it says a gate may be set and how to check.
+func TestWatchVersionGatedDiagnostics(t *testing.T) {
+ const host, hash = "report.bab.test", "$2a$10$stand-in-hash"
+ srv := gatedEdge(t, host, hash, "v2")
+ for _, c := range []struct {
+ name string
+ target watchTarget
+ want string
+ }{
+ {"refused pass", watchTarget{Host: host, Origin: srv.URL, Cookie: passFor(host, "$2a$10$old-password"), Gated: true},
+ "the password gate refused the watch's pass (HTTP 401)"},
+ {"no pass", watchTarget{Host: host, Origin: srv.URL},
+ "HTTP 401 with no version header. a password gate may be set on this host"},
+ } {
+ err := watchVersion(&bytes.Buffer{}, []watchTarget{c.target}, "v2", 50*time.Millisecond, 10*time.Millisecond, nil)
+ if err == nil {
+ t.Fatalf("%s: must time out", c.name)
+ }
+ if !strings.Contains(err.Error(), c.want) {
+ t.Errorf("%s: report = %v, want it to contain %q", c.name, err, c.want)
+ }
+ if strings.Contains(err.Error(), "failing to wake") {
+ t.Errorf("%s: a gate's 401 blamed on a wake: %v", c.name, err)
+ }
+ }
+}
+
+// TestWatchVersionAppOwn401IsNotTheGate: an app's own 401 carries the
+// version header (the edge stamps it before proxying), so it is judged by
+// the ordinary rules and never mistaken for the gate.
+func TestWatchVersionAppOwn401IsNotTheGate(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Carlos-Version", "v2")
+ w.WriteHeader(http.StatusUnauthorized)
+ }))
+ defer srv.Close()
+ err := watchVersion(&bytes.Buffer{}, []watchTarget{{Host: "x.test", Origin: srv.URL}}, "v2",
+ 50*time.Millisecond, 10*time.Millisecond, nil)
+ if err == nil || !strings.Contains(err.Error(), "serving v2 but HTTP 401") {
+ t.Fatalf("an app's own 401 must read as the app failing: %v", err)
+ }
+}
+
+// TestPassGatedTargets: open hosts pass through untouched and never cost a
+// console call; a gated host gets its pass; a host the console REFUSES a
+// pass for leaves the watch with a line naming the host and why; a host
+// whose gate was cleared meanwhile is watched as open; and a console that
+// merely FAILED keeps the host in the watch, pass-less, so a blip can never
+// turn into a green deploy nobody checked.
+func TestPassGatedTargets(t *testing.T) {
+ targets := []watchTarget{{Host: "open.test"}, {Host: "gated.test"}, {Host: "nopass.test"},
+ {Host: "cleared.test"}, {Host: "busy.test"}}
+ gated := map[string]bool{"gated.test": true, "nopass.test": true, "cleared.test": true, "busy.test": true}
+ var asked []string
+ pass := func(host string) (string, error) {
+ asked = append(asked, host)
+ switch host {
+ case "nopass.test":
+ return "", &gatePassRefused{errors.New("404 page not found")}
+ case "cleared.test":
+ return "", errGateOpen
+ case "busy.test":
+ return "", errors.New("too many instance requests")
+ }
+ return "carlos_gate=v1.1.x", nil
+ }
+ var out bytes.Buffer
+ got := passGatedTargets(&out, targets, gated, "v2", "edge", pass)
+ var hosts []string
+ for _, g := range got {
+ hosts = append(hosts, g.Host)
+ }
+ if strings.Join(hosts, ",") != "open.test,gated.test,cleared.test,busy.test" {
+ t.Fatalf("watched %v, want every host but nopass.test", hosts)
+ }
+ if got[0].Cookie != "" || got[0].Gated {
+ t.Errorf("open host picked up a pass: %+v", got[0])
+ }
+ if got[1].Cookie != "carlos_gate=v1.1.x" || !got[1].Gated {
+ t.Errorf("gated host = %+v, want its pass", got[1])
+ }
+ for _, g := range got[2:] {
+ if g.Cookie != "" || g.Gated {
+ t.Errorf("%s = %+v, want it watched with no pass", g.Host, g)
+ }
+ }
+ if strings.Join(asked, ",") != "gated.test,nopass.test,cleared.test,busy.test" {
+ t.Errorf("asked the console for %v, want only the gated hosts", asked)
+ }
+ for _, want := range []string{
+ "gated.test has a password gate. the watch checks the version",
+ "nopass.test has a password gate and the console could not give the watch a pass: 404 page not found",
+ "v2 is on edge. its version can't be checked from outside, so this host is not watched",
+ "busy.test has a password gate and the console could not give the watch a pass: too many instance requests\n watching it anyway",
+ } {
+ if !strings.Contains(out.String(), want) {
+ t.Errorf("output lacks %q:\n%s", want, out.String())
+ }
+ }
+}
+
+// TestInstanceGatePassClassifiesAnswers: the console's answers map onto the
+// three outcomes passGatedTargets acts on, and a 200 comes back as a Cookie
+// header value ready to send.
+func TestInstanceGatePassClassifiesAnswers(t *testing.T) {
+ for _, c := range []struct {
+ status int
+ body string
+ check func(cookie string, err error) bool
+ }{
+ {http.StatusOK, `{"name":"carlos_gate","value":"v1.9.abc","expires":"2026-09-26T21:00:00Z"}`,
+ func(c string, err error) bool { return err == nil && c == "carlos_gate=v1.9.abc" }},
+ {http.StatusConflict, `{"error":"no password gate is set on h.test"}`,
+ func(_ string, err error) bool { return errors.Is(err, errGateOpen) }},
+ {http.StatusNotFound, "404 page not found\n",
+ func(_ string, err error) bool { var r *gatePassRefused; return errors.As(err, &r) }},
+ {http.StatusForbidden, `{"error":"this credential holds role \"operate\""}`,
+ func(_ string, err error) bool { var r *gatePassRefused; return errors.As(err, &r) }},
+ {http.StatusTooManyRequests, `{"error":"too many instance requests"}`,
+ func(_ string, err error) bool {
+ var r *gatePassRefused
+ return err != nil && !errors.As(err, &r) && !errors.Is(err, errGateOpen)
+ }},
+ {http.StatusOK, `{}`,
+ func(_ string, err error) bool { return err != nil }},
+ } {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost || !strings.HasSuffix(r.URL.Path, "/instances/h.test/gate-pass") {
+ t.Errorf("request = %s %s", r.Method, r.URL.Path)
+ }
+ w.WriteHeader(c.status)
+ w.Write([]byte(c.body))
+ }))
+ sess := &apiSession{c: newConsoleClient(ts.URL, "tok"), console: ts.URL}
+ cookie, err := sess.instanceGatePass(context.Background(), apiAcct{Seg: "bab"}, "report", "h.test")
+ ts.Close()
+ if !c.check(cookie, err) {
+ t.Errorf("status %d: got (%q, %v)", c.status, cookie, err)
+ }
+ }
+}