d2ae23a51 · carlos/platform

diff --git a/cli.go b/cli.go

index bf6ab5830..85bfd7b5b 100644

--- a/cli.go

+++ b/cli.go

@@ -391,6 +391,9 @@ func runInstances(args []string) {

case "set-environment":

runInstancesSetEnvironment(args[1:])

return

+ case "retry":

+ runInstancesRetry(args[1:])

+ return

}

fs := flag.NewFlagSet("instances", flag.ExitOnError)

diff --git a/instances_cli.go b/instances_cli.go

index 2bdaa3963..da4bc7aed 100644

--- a/instances_cli.go

+++ b/instances_cli.go

@@ -23,10 +23,13 @@ package main

import (

"context"

+ "encoding/json"

"errors"

"flag"

"fmt"

"log"

+ "net/http"

+ "net/url"

"os"

"strings"

@@ -206,6 +209,8 @@ func instancesEnable(ctx context.Context, s release.Store, acct, app string, m p

return provision.PutManifest(ctx, s, acct, app, m)

}

+// TODO-COPY zdd.instances-retry.usage-line

+// TODO-COPY zdd.instances-retry.usage-para

const instancesRecordsUsage = `usage: carlos instances create --app <a> --host <h> [--region <r>] [--channel <c>]

[--kind instance|service|static] [--environment <e>]

[--addr <a:p,...>] [--health-path <p>] [--health-status <n>]

@@ -215,6 +220,7 @@ const instancesRecordsUsage = `usage: carlos instances create --app <a> --host <

[--health-path <p>] [--health-status <n>]

carlos instances set-channel --app <a> --host <h> --channel <c>

carlos instances set-environment --app <a> --host <h> --environment <e>

+ carlos instances retry --app <a> <host>

Signed-in only (carlos auth login): instance records are console-authorized

writes — the console checks the host against the app's manifest and the

@@ -233,6 +239,10 @@ pass — so this is the verb that changes it, not "carlos route --channel",

which edits a row the next reconcile overwrites. A move that the app's

pipeline gates is proposed and finished in the console, like a promote.

+retry clears the failure hold on one instance: the build that failed on its

+box is tried again there, as a canary. "carlos promote --retry <version>

+<channel>" does the same on every box.

+

--environment binds a route to one of the app's named config bundles

("carlos env environments" lists them; "carlos env --environment <e>" writes

them). Omitted, a route serves the app's default bundle — which matters most

@@ -895,3 +905,76 @@ func runInstancesSetEnvironment(args []string) {

// "it didn't work" and "it needs a restart".

mutedf(stdout, " the binding lands within seconds; restart the instance to pick the new values up: carlos restart --app %s\n", app2)

}

+

+const (

+ // TODO-COPY zdd.instances-retry.done

+ instancesRetryDone = "retry requested for %s: its next attempt at %s runs as a canary"

+ // TODO-COPY zdd.instances-retry.old-console

+ instancesRetryOldConsole = "not found: either %s has no instance %s, or this console predates carlos instances retry (deploy a console built from current main)"

+)

+

+type apiRetryResult struct {

+ ID string `json:"id"`

+ Version string `json:"version"`

+ Host string `json:"host"`

+}

+

+// instanceRetry is `carlos instances retry`'s request. A bare 404 is "not

+// yours" or a console from before the door; the message names both.

+func (s *apiSession) instanceRetry(ctx context.Context, a apiAcct, app, host string, messages ...string) (apiRetryResult, error) {

+ in := struct {

+ Message string `json:"message,omitempty"`

+ }{firstMutationMessage(messages)}

+ status, body, err := s.postJSON(ctx, apiPath(a, app, "/instances/"+url.PathEscape(host)+"/retry"), in)

+ if err != nil {

+ return apiRetryResult{}, err

+ }

+ if status != http.StatusOK {

+ if status == http.StatusNotFound && !hasJSONError(body) {

+ return apiRetryResult{}, fmt.Errorf(instancesRetryOldConsole, app, host)

+ }

+ return apiRetryResult{}, apiErrorFromBody(status, body)

+ }

+ var out apiRetryResult

+ if err := json.Unmarshal(body, &out); err != nil {

+ return apiRetryResult{}, fmt.Errorf("instances retry: unreadable response: %w", err)

+ }

+ return out, nil

+}

+

+// runInstancesRetry is `carlos instances retry <host>` (spec §8.4, §9.1).

+// The host may come first, as the spec writes it, or after the flags.

+func runInstancesRetry(args []string) {

+ host, args := leadingArg(args)

+ fs := flag.NewFlagSet("instances retry", flag.ExitOnError)

+ messageFlags := addMutationMessageFlags(fs)

+ app := appFlag(fs, "app the instance belongs to")

+ consoleFlag := consoleOption(fs, "console to act through (default: the only logged-in one)")

+ acctRaw := fs.String("account", "", "account (sqid or name) the app belongs to (default: your only account, or the configured/chosen one)")

+ acctIDRaw := fs.String("account-id", "", "account sqid, matched as a sqid ONLY — supersedes --account")

+ fs.Usage = flagUsage(fs, instancesRecordsUsage)

+ fs.Parse(args)

+ switch {

+ case host == "" && fs.NArg() == 1:

+ host = strings.TrimSpace(fs.Arg(0))

+ case fs.NArg() != 0:

+ log.Fatal(instancesRecordsUsage)

+ }

+ if host == "" {

+ log.Fatal(instancesRecordsUsage)

+ }

+ host = strings.ToLower(host)

+ sess := requireAPISession("retry", *consoleFlag)

+ printUsing(stdout, sess)

+ acct := resolvedAccount(sess, acctRaw, acctIDRaw)

+ app2 := resolvedApp(sess, acct, *app, instancesRecordsUsage)

+ message, err := resolveMutationMessage(fs, messageFlags)

+ if err != nil {

+ log.Fatalf("instances retry: %v", err)

+ }

+ res, err := sess.instanceRetry(context.Background(), acct, app2, host, message)

+ if err != nil {

+ log.Fatalf("instances retry: %v", err)

+ }

+ fmt.Fprintf(stdout, instancesRetryDone+"\n", host, res.Version)

+}

diff --git a/instances_e2e_test.go b/instances_e2e_test.go

index 01ba884c7..0bb510e61 100644

--- a/instances_e2e_test.go

+++ b/instances_e2e_test.go

@@ -14,6 +14,7 @@ package main

import (

"context"

+ "fmt"

"net/http"

"net/http/httptest"

"path/filepath"

@@ -21,9 +22,11 @@ import (

"testing"

"time"

+ "github.com/carlosframework/platform/internal/appconfig"

"github.com/carlosframework/platform/internal/console"

"github.com/carlosframework/platform/internal/provision"

"github.com/carlosframework/platform/internal/release"

+ "github.com/carlosframework/platform/internal/status"

)

func TestE2EInstancesThroughConsole(t *testing.T) {

@@ -275,6 +278,39 @@ func TestE2EInstancesThroughConsole(t *testing.T) {

// apimode_test.go), which is where the request/response round trip is

// actually observable.

+ // --- apps set (zero-downtime plan 6, Task 5), in the spec's order --------

+ buf.Reset()

+ runAppsSet([]string{"titogo", "--swap", "hold", "-account", "ops"})

+ if want := fmt.Sprintf(appsSetDone, "titogo", "hold", 1); !strings.Contains(buf.String(), want) {

+ t.Fatalf("apps set stdout = %q, want %q", buf.String(), want)

+ }

+ policy, err := appconfig.GetSwapPolicy(context.Background(), relStore, release.OpsAccount, "titogo")

+ if err != nil || policy.Swap != appconfig.SwapHold {

+ t.Fatalf("stored policy = %+v, %v", policy, err)

+ }

+

+ // --- instances retry (Task 11): host first, as the spec writes it --------

+ box := status.CollectBox(nil, status.Probes{MachineID: "e2e-box"}, time.Now())

+ box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{

+ haHost: {Account: release.OpsAccount, App: "titogo", Installed: "v2", Serving: "v1", Good: "v1",

+ Awake: true, HeldBy: "v2", Reason: status.ReasonHealth},

+ }}}

+ if err := status.WriteBox(context.Background(), relStore, box); err != nil {

+ t.Fatal(err)

+ }

+ buf.Reset()

+ runInstancesRetry([]string{haHost, "-app", "titogo", "-account", "ops"})

+ if want := fmt.Sprintf(instancesRetryDone, haHost, "v2"); !strings.Contains(buf.String(), want) {

+ t.Fatalf("instances retry stdout = %q, want %q", buf.String(), want)

+ }

+ scopes, found, err := release.LoadRestartScopes(context.Background(), relStore, release.OpsAccount, "titogo")

+ if err != nil || !found || len(scopes.Requests) == 0 {

+ t.Fatalf("restart scopes = %+v, %v, %v", scopes, found, err)

+ }

+ if last := scopes.Requests[len(scopes.Requests)-1]; last.Scope.Kind != release.RestartScopeRetry || last.Scope.Value != haHost {

+ t.Fatalf("last request = %+v", last)

+ }

+

// The whole flow ran through the console: nothing here may have touched

// a local registry.

checkNoRegistry()

diff --git a/internal/console/authcli.go b/internal/console/authcli.go

index 99eb8f66c..86e75b9bf 100644

--- a/internal/console/authcli.go

+++ b/internal/console/authcli.go

@@ -685,6 +685,9 @@ func (s *Server) registerCLIRoutes() {

s.registerMovesPhase1Routes()

}

s.cliRoute("DELETE /api/cli/apps/{account}/{app}/instances/{host}", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceDelete)

+ // `carlos instances retry <host>` (apiretry.go): the operate family's, as

+ // restart is. The {host} wildcard and the 404 are the instance family's.

+ s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/retry", svcrole.Allow(svcrole.Operate), s.handleAPIInstanceRetry)

// The `carlos db` replica door (apidb.go; spec 2026-09-02 §6): an

// instance's replicated LTX files, by coordinate, for a member to

// restore on their own machine. The zero Set, deliberately: the payload

diff --git a/internal/console/claim_invariant_fixtures_test.go b/internal/console/claim_invariant_fixtures_test.go

index 7b413dd46..ec5ef687e 100644

--- a/internal/console/claim_invariant_fixtures_test.go

+++ b/internal/console/claim_invariant_fixtures_test.go

@@ -38,6 +38,7 @@ import (

"github.com/carlosframework/platform/internal/registry"

"github.com/carlosframework/platform/internal/relay"

"github.com/carlosframework/platform/internal/release"

+ "github.com/carlosframework/platform/internal/status"

)

// ---- table helpers ----------------------------------------------------------

@@ -260,6 +261,22 @@ func (cx *claimCase) instanceWorld(t *testing.T) {

enableInstances(t, cx.consoleFixture, "titogo", titoManifest())

}

+// heldInstance is the host-retry door's world: one instance whose box reports

+// it held back by a failed build.

+func (cx *claimCase) heldInstance(t *testing.T, host string) {

+ t.Helper()

+ cx.instanceWorld(t)

+ cx.instance(t, `{"host":"`+host+`","region":"ie","channel":"edge"}`)

+ box := status.CollectBox(nil, status.Probes{MachineID: "claim-box"}, time.Now())

+ box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{

+ host: {Account: "bab", App: "titogo", Installed: "v2", Serving: "v1", Good: "v1", Awake: true,

+ HeldBy: "v2", Reason: status.ReasonHealth},

+ }}}

+ if err := status.WriteBox(context.Background(), cx.rel, box); err != nil {

+ t.Fatal(err)

+ }

+}

+

func (cx *claimCase) instance(t *testing.T, body string) {

t.Helper()

cx.mustAPI(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", body)

@@ -1193,6 +1210,30 @@ var claimFixtures = map[string]claimFixture{

status: http.StatusConflict,

msg: claimMoveUnsupportedMsg,

}},

+ "route POST /api/cli/apps/{account}/{app}/instances/{host}/retry": {door: &claimDoor{

+ apps: claimRefs("bab/titogo"),

+ setup: func(t *testing.T, cx *claimCase) { cx.heldInstance(t, "web.go.tito.io") },

+ do: claimAPI(http.MethodPost, "/api/cli/apps/bab/titogo/instances/web.go.tito.io/retry", `{}`),

+ check: func(t *testing.T, cx *claimCase, open bool) string {

+ g, found, err := release.LoadRestartScopes(context.Background(), cx.rel, "bab", "titogo")

+ if err != nil {

+ return err.Error()

+ }

+ asked := false

+ if found {

+ for _, r := range g.Requests {

+ asked = asked || (r.Scope.Kind == release.RestartScopeRetry && r.Scope.Value == "web.go.tito.io")

+ }

+ switch {

+ case open && !asked:

+ return "the retry request was not written: the handler never reached the narrowed log"

+ case !open && asked:

+ return "a retry request was written while the app was closed under a foreign exclusive operation"

+ }

+ return ""

+ },

+ }},

"route POST /api/cli/apps/{account}/{app}/keep": {door: &claimDoor{

apps: claimRefs("bab/tmp-claim-keep"),

setup: func(t *testing.T, cx *claimCase) { cx.temporary(t, "tmp-claim-keep") },

diff --git a/internal/console/testdata/claim-routes.txt b/internal/console/testdata/claim-routes.txt

index 01bf8648d..f05179dc1 100644

--- a/internal/console/testdata/claim-routes.txt

+++ b/internal/console/testdata/claim-routes.txt

@@ -175,6 +175,7 @@ route POST /api/cli/apps/{account}/{app}/instances

route POST /api/cli/apps/{account}/{app}/instances/{host}/channel

route POST /api/cli/apps/{account}/{app}/instances/{host}/move

route POST /api/cli/apps/{account}/{app}/instances/{host}/move/cut

+route POST /api/cli/apps/{account}/{app}/instances/{host}/retry

route POST /api/cli/apps/{account}/{app}/keep

route POST /api/cli/apps/{account}/{app}/ledgers/publish

route POST /api/cli/apps/{account}/{app}/ledgers/{ledger}/append

diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go

index 5237a17b4..38319e017 100644

--- a/internal/provision/plan9_b8_census_test.go

+++ b/internal/provision/plan9_b8_census_test.go

@@ -487,7 +487,8 @@ var plan9B8CensusInventory = []censusEntry{

{"apimode.go", "apiSession.sidecarGet", "key-site", "lit:sidecars", "1a35497dbb705df114430141c124296fb6be30690c07eeedb50ee3cc2b33de93"},

{"apimode.go", "apiSession.sidecarDelete", "key-site", "lit:sidecars", "f23690fec191a001df364d3f39d06d3c287ce95f8a2b1b4ea94306d3536fbb3c"},

{"apimode.go", "apiSession.sidecarWake", "key-site", "lit:sidecars", "9d1239f5d3c2977de9f406ede15ca97756ff2929268857047a58c28d344d9868"},

- {"cli.go", "runInstances", "key-site", "lit:instances", "cbd14f548169ef95e10a75ed272c9178ed687f211c55f4152f638d11a8c91a5f"},

+ {"cli.go", "runInstances", "key-site", "lit:instances", "8da9ea68dd9a62b21d41ff3a61362cb85020f9336728c51baeafd524b804c3f1"},

+ {"instances_cli.go", "apiSession.instanceRetry", "key-site", "lit:instances", "68e68933d4817ed2191cefa45bb63e70b2d71bdf7acec76aeb79ce8f07245b6e"},

// Names instance record paths in refusals; writes local registry rows, not provision objects.

{"cli.go", "runRoute", "key-site", "lit:instances", "36862b228fb5144bb9a282274f957c5479ed23dd6965673e88c09cc9e97fc66a"},

{"cli.go", "runRouteHibernate", "key-site", "key:SidecarManifestKey", "c151cae2cbc57943e9a5d0a5dac611823d769a6b651d5dbc2fe86182f6e098b1"},

@@ -570,7 +571,7 @@ var plan9B8CensusInventory = []censusEntry{

{"internal/console/apisystemupdate.go", "Server.mirrorSystemArtifact", "nonrecord", "sel:Put", "76a8dcb558e3723ad5774788da37c22234c229dc3257ded5a6c7b04746c69c2f"},

{"internal/console/appsettings.go", "Server.handleAppGate", "record-writer", "key:UpdateRecord", "011431f5459529c0abf354edc2cb772cb6a7f2816d3bb50d91598d7d241838ce"},

// Registers instance and sidecar HTTP paths; record handlers use the reviewed provision writers.

- {"internal/console/authcli.go", "Server.registerCLIRoutes", "key-site", "lit:instances,lit:sidecars", "359b8c8f37aed115048ca82cf300bb7082a806fd22a07e1ee98dc6a5663461fe"},

+ {"internal/console/authcli.go", "Server.registerCLIRoutes", "key-site", "lit:instances,lit:sidecars", "eb6388aed42540177e6b1531fb6e79ed4d66127ec8774c8fb68858ffd0c79296"},

{"internal/console/browserrestart.go", "Server.handleBrowserRestart", "key-site", "lit:instances", "955d71813504f2cfec1e1f64817d5aa5a921aa38e1044c0e903d2cfbe4d073f4"},

{"internal/console/channelmove.go", "Server.handleAPIInstanceChannel", "record-writer", "key:UpdateRecord", "91a2b3157bde9af832e8d95b17d0c384910229982945818f8b52f62dafddb342"},

{"internal/console/channelmove.go", "Server.executePendingRouteChannel", "record-writer", "key:UpdateRecord", "b3b9a12213261a70b680ad5290957ebe7434eff633a73c2d80c75cc24d026d9a"},

carlos / platform Public

WIP: stopped at the weekly usage limit