amadan

carlos / platform Public

zdd-6 task 11: carlos instances retry, door and CLI

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 7401745ef1a32301cfa55a496f3458723eb35ac8 parent d2ae23a
4 files changed, +306 −0
  • instances_retry_test.go +37 −0
  • internal/console/apiretry.go +123 −0
  • internal/console/apiretry_test.go +97 −0
  • internal/host/restartscope_test.go +49 −0
diff --git a/instances_retry_test.go b/instances_retry_test.go
new file mode 100644
index 000000000..cc02dd9a5
--- /dev/null
+++ b/instances_retry_test.go
@@ -0,0 +1,37 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func TestAPISessionInstanceRetryWireShape(t *testing.T) {
+ var path, body string
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ b, _ := io.ReadAll(r.Body)
+ path, body = r.URL.Path, string(b)
+ w.Header().Set("Content-Type", "application/json")
+ io.WriteString(w, `{"result":"requested","id":"r1","version":"v2","host":"a.shop.test"}`)
+ }))
+ defer ts.Close()
+ res, err := newTestSession(ts.URL).instanceRetry(context.Background(), apiAcct{Seg: "bab"}, "shop", "a.shop.test")
+ if err != nil || res.Version != "v2" || res.ID != "r1" {
+ t.Fatalf("instanceRetry = %+v, %v", res, err)
+ }
+ if path != "/api/cli/apps/bab/shop/instances/a.shop.test/retry" || body != `{}` {
+ t.Fatalf("sent %s %s", path, body)
+ }
+}
+
+func TestAPISessionInstanceRetryOldConsole(t *testing.T) {
+ ts := httptest.NewServer(http.NotFoundHandler())
+ defer ts.Close()
+ _, err := newTestSession(ts.URL).instanceRetry(context.Background(), apiAcct{Seg: "bab"}, "shop", "a.shop.test")
+ if err == nil || err.Error() != fmt.Sprintf(instancesRetryOldConsole, "shop", "a.shop.test") {
+ t.Fatalf("err = %v", err)
+ }
+}
diff --git a/internal/console/apiretry.go b/internal/console/apiretry.go
new file mode 100644
index 000000000..1b403d1c3
--- /dev/null
+++ b/internal/console/apiretry.go
@@ -0,0 +1,123 @@
+// The host-retry door (zero-downtime spec §8.4, §9.1): `carlos instances
+// retry <host>` clears one instance's failure hold, and its next attempt
+// runs as a canary. The app-wide retry is `carlos promote --retry`, which
+// re-signs the channel pointer; this door has no signing to do.
+//
+// The request rides the narrowed-restart log as selector kind "retry"
+// (release.RestartScopeRetry). A box built before that kind steps over it;
+// one built after hands it to its rollout manager. ROLE: operate, as
+// `carlos restart` is.
+package console
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log"
+ "net/http"
+ "os"
+ "time"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+const (
+ // TODO-COPY zdd.api.retry-store
+ apiRetryStore = "could not write the retry request. Try again"
+ // TODO-COPY zdd.api.retry-host-audit
+ apiRetryHostAudit = "requested a retry of %s on %s (%s)"
+ // TODO-COPY zdd.api.retry-not-held
+ apiRetryNotHeld = "%s is not held back by a failed build, as far as its box last reported, so there is nothing to retry"
+)
+
+var (
+ errRetryNoRecord = errors.New("retry: no such instance")
+ errRetryNotHeld = errors.New("retry: not held")
+)
+
+// retryHostBuild is the build holding host back, as its box reports it. The
+// record must exist in this account and app (the instance family's 404), and
+// the report must be this app's own (boxJoin.rollout). A box that could not
+// arm its scheduler reports no hold, so its host is not held.
+func (s *Server) retryHostBuild(ctx context.Context, acct *Account, app, host string) (string, error) {
+ if _, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host); err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return "", errRetryNoRecord
+ }
+ return "", err
+ }
+ ro := loadBoxJoin(ctx, s.cfg.Releases, time.Now()).rollout(host, acct.Sqid, app)
+ if ro == nil || ro.Unavailable != "" || ro.HeldBy == "" {
+ return "", errRetryNotHeld
+ }
+ return ro.HeldBy, nil
+}
+
+// requestHostRetry appends the retry request, by the same append the
+// narrowed-restart door uses, and announces it on the serial.
+func (s *Server) requestHostRetry(ctx context.Context, acct, app, host string) (release.RestartRequest, error) {
+ req, err := release.AppendRestartScope(ctx, s.cfg.Releases, acct, app,
+ release.RestartScope{Kind: release.RestartScopeRetry, Value: host})
+ if err != nil {
+ return release.RestartRequest{}, err
+ }
+ s.bumpSerial(ctx)
+ return req, nil
+}
+
+func (s *Server) handleAPIInstanceRetry(w http.ResponseWriter, r *http.Request) {
+ addr, acct, app, ok := s.apiAccountApp(w, r)
+ if !ok {
+ return
+ }
+ host := r.PathValue("host")
+ if !provision.ValidHostShape(host) {
+ http.NotFound(w, r)
+ return
+ }
+ p, _ := principalFrom(r.Context())
+ if !s.allowWrite(p, time.Now()) {
+ jsonError(w, http.StatusTooManyRequests, "too many requests. Wait a minute and try again")
+ return
+ }
+ if s.cfg.Releases == nil {
+ jsonError(w, http.StatusServiceUnavailable,
+ "no deployment bucket on this host. This console cannot request a retry")
+ return
+ }
+ var req struct {
+ Message string `json:"message"`
+ }
+ if !readJSON(w, r, &req) {
+ return
+ }
+ message, ok := checkedActivityMessage(w, req.Message)
+ if !ok {
+ return
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
+ defer cancel()
+ build, err := s.retryHostBuild(ctx, acct, app, host)
+ switch {
+ case errors.Is(err, errRetryNoRecord):
+ http.NotFound(w, r)
+ return
+ case errors.Is(err, errRetryNotHeld):
+ jsonError(w, http.StatusConflict, fmt.Sprintf(apiRetryNotHeld, host))
+ return
+ case err != nil:
+ log.Printf("console: api instance retry %s/%s/%s: %v", acct.Sqid, app, host, err)
+ jsonError(w, http.StatusBadGateway, "could not read the instance")
+ return
+ }
+ rr, err := s.requestHostRetry(ctx, acct.Sqid, app, host)
+ if err != nil {
+ log.Printf("console: api instance retry %s/%s/%s: %v", acct.Sqid, app, host, err)
+ jsonError(w, http.StatusBadGateway, apiRetryStore)
+ return
+ }
+ s.auditActivity(acct, app, addr, Activity{Action: "retry", TargetType: "instance", Target: host,
+ Message: message, Detail: fmt.Sprintf(apiRetryHostAudit, build, host, "cli")})
+ writeJSON(w, http.StatusOK, map[string]any{"result": "requested", "id": rr.ID, "version": build, "host": host})
+}
diff --git a/internal/console/apiretry_test.go b/internal/console/apiretry_test.go
new file mode 100644
index 000000000..d113db12e
--- /dev/null
+++ b/internal/console/apiretry_test.go
@@ -0,0 +1,97 @@
+package console
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+// lastScoped reads the newest request in the app's narrowed log.
+func lastScoped(t *testing.T, fx *consoleFixture, app string) release.RestartRequest {
+ t.Helper()
+ g, found, err := release.LoadRestartScopes(context.Background(), fx.rel, fx.sqid, app)
+ if err != nil || !found || len(g.Requests) == 0 {
+ t.Fatalf("restart scopes = %+v, %v, %v", g, found, err)
+ }
+ return g.Requests[len(g.Requests)-1]
+}
+
+func writeHeldBox(t *testing.T, fx *consoleFixture, host, app, build string) {
+ t.Helper()
+ box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now())
+ box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{
+ host: {Account: fx.sqid, App: app, Installed: build, Serving: "v1", Good: "v1", Awake: true, HeldBy: build, Reason: status.ReasonHealth},
+ }}}
+ if err := status.WriteBox(context.Background(), fx.rel, box); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestAPIInstanceRetryWritesARetryRequest(t *testing.T) {
+ fx := newConsoleFixture(t)
+ const host = "web.kass.carlosapp.dev"
+ sibling(t, fx, "kass", host, "stable")
+ token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry")
+ u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry"
+
+ if code, _, body := rawReq(t, http.MethodPost, u, token, `{}`); code != http.StatusConflict {
+ t.Fatalf("no report = %d %s, want 409", code, body)
+ }
+ writeHeldBox(t, fx, host, "kass", "v2")
+ code, _, body := rawReq(t, http.MethodPost, u, token, `{}`)
+ if code != http.StatusOK {
+ t.Fatalf("retry = %d %s", code, body)
+ }
+ var out struct{ ID, Version, Host string }
+ if err := json.Unmarshal([]byte(body), &out); err != nil || out.Version != "v2" || out.Host != host || out.ID == "" {
+ t.Fatalf("answer = %s", body)
+ }
+ if r := lastScoped(t, fx, "kass"); r.Scope.Kind != release.RestartScopeRetry || r.Scope.Value != host || r.ID != out.ID {
+ t.Fatalf("stored = %+v", r)
+ }
+ if n := serial(t, fx); n != 1 {
+ t.Fatalf("serial = %d", n)
+ }
+ code, _, body = rawReq(t, http.MethodPost, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/kass/instances/nope.kass.carlosapp.dev/retry", token, `{}`)
+ if code != http.StatusNotFound || body != "404 page not found\n" {
+ t.Fatalf("unknown host = %d %q", code, body)
+ }
+}
+
+func TestAPIInstanceRetryIgnoresAnotherAppsReport(t *testing.T) {
+ fx := newConsoleFixture(t)
+ const host = "web.kass.carlosapp.dev"
+ sibling(t, fx, "kass", host, "stable")
+ writeHeldBox(t, fx, host, "otherapp", "z2")
+ token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-x")
+ u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry"
+ if code, _, body := rawReq(t, http.MethodPost, u, token, `{}`); code != http.StatusConflict {
+ t.Fatalf("another app's hold = %d %s, want 409", code, body)
+ }
+}
+
+// A box that could not arm its scheduler reports no per-host state; the host is
+// not held, whatever else its view says.
+func TestAPIInstanceRetryRefusesAnUnavailableBox(t *testing.T) {
+ fx := newConsoleFixture(t)
+ const host = "web.kass.carlosapp.dev"
+ sibling(t, fx, "kass", host, "stable")
+ box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now())
+ box.Metrics = &status.BoxMetrics{
+ RouteVersions: map[string]string{host: "v2"},
+ Rollout: &status.RolloutReport{Unavailable: "no-deploy-store"},
+ }
+ if err := status.WriteBox(context.Background(), fx.rel, box); err != nil {
+ t.Fatal(err)
+ }
+ token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-u")
+ u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry"
+ if code, _, body := rawReq(t, http.MethodPost, u, token, `{}`); code != http.StatusConflict {
+ t.Fatalf("unavailable box = %d %s, want 409", code, body)
+ }
+}
diff --git a/internal/host/restartscope_test.go b/internal/host/restartscope_test.go
index 0829145e3..b4fba8244 100644
--- a/internal/host/restartscope_test.go
+++ b/internal/host/restartscope_test.go
@@ -631,3 +631,52 @@ func TestRestartLegDropsARetryWithNoManager(t *testing.T) {
t.Fatalf("the retry was seen again on the next pass: %q", got)
}
}
+
+// A retry names a host but is not a restart: the narrowing that resolves hosts
+// must never address one for it, on the agent's half or the root pass's.
+func TestScopedHostsAndRootPassStepOverARetry(t *testing.T) {
+ routes := titogoRoutes()
+ var logbuf bytes.Buffer
+ log.SetOutput(&logbuf)
+ defer log.SetOutput(os.Stderr)
+ got := scopedHosts(routes, [2]string{"bab", "titogo"}, []release.RestartRequest{
+ {ID: "1", Scope: release.RestartScope{Kind: release.RestartScopeRetry, Value: "bill.example.com"}},
+ }, func(registry.Route) bool { return true })
+ if len(got) != 0 {
+ t.Fatalf("a retry addressed %v for a restart", got)
+ }
+ if strings.Contains(logbuf.String(), "does not understand") {
+ t.Fatalf("a retry was logged as an unknown selector: %q", logbuf.String())
+ }
+
+ ctx, s := genKit(t)
+ marks := restartMarksPath(t.TempDir())
+ restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn) // baseline
+ askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeRetry, "bill.example.com")
+ rec := &recordingRestart{}
+ restartPass(ctx, routes, s, marks, rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("the root pass cycled %v for a retry", rec.hosts)
+ }
+}
+
+// A retry and a real narrowed restart in one poll window: the retry goes to
+// the manager, only the restart cycles, and root is asked for its pass.
+func TestRestartLegSeparatesARetryFromARestartInOneWindow(t *testing.T) {
+ ctx, s := genKit(t)
+ dd := t.TempDir()
+ routes := []registry.Route{execRoute("a.example.com", "bab", "eleven"), execRoute("b.example.com", "bab", "eleven")}
+ mem := newRestartMemory()
+ var retried, bounced []string
+ mem.retryHost = func(h string) bool { retried = append(retried, h); return true }
+ bounce := func(h string) bool { bounced = append(bounced, h); return true }
+ restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) // baseline
+ askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeRetry, "a.example.com")
+ askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeHost, "b.example.com")
+ if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); !asked {
+ t.Fatal("the restart in the window did not ask for a root pass")
+ }
+ if !slices.Equal(retried, []string{"a.example.com"}) || !slices.Equal(bounced, []string{"b.example.com"}) {
+ t.Fatalf("retried %v, bounced %v; want the retry to a.example.com and the bounce of b.example.com only", retried, bounced)
+ }
+}