| index 5ff3c017c..e82b49427 100644 |
| --- a/system.go |
| +++ b/system.go |
| @@ -1,9 +1,18 @@ |
| package main |
| |
| import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "flag" |
| "fmt" |
| + "io" |
| "log" |
| + "mime/multipart" |
| + "net/http" |
| "os" |
| + "strings" |
| + "time" |
| |
| "github.com/carlosframework/platform/internal/host" |
| ) |
| @@ -44,14 +53,30 @@ func runSystem(args []string) { |
| case "help", "-h", "--help": |
| systemUsage() |
| default: |
| - if len(args) > 0 && verb == "" { |
| - fmt.Fprintf(os.Stderr, "system: unknown verb %q\n\n", args[0]) |
| + if msg := systemUnknownVerbMessage(args, rest); msg != "" { |
| + fmt.Fprintf(os.Stderr, "%s\n\n", msg) |
| } |
| systemUsage() |
| os.Exit(2) |
| } |
| } |
| |
| +// systemUnknownVerbMessage is the line above the usage when no verb matched. |
| +// It reads REST, not args, because systemVerb has already eaten the optional |
| +// component: `carlos system router console` is an unknown verb "console", |
| +// never an unknown verb "router". "" means say nothing at all — bare |
| +// `carlos system` is someone asking for the usage, not a typo to report. |
| +func systemUnknownVerbMessage(args, rest []string) string { |
| + switch { |
| + case len(rest) > 0: |
| + return fmt.Sprintf("system: unknown verb %q", rest[0]) |
| + case len(args) > 0: |
| + return "system: no verb given" |
| + default: |
| + return "" |
| + } |
| +} |
| + |
| func systemUsage() { |
| fmt.Fprint(os.Stderr, `usage: carlos system <verb> [flags] |
| carlos system router <verb> [flags] |
| @@ -65,9 +90,332 @@ The platform is not an app: the console is updated with carlos deploy --app cons |
| `) |
| } |
| |
| -// Placeholders until Task 12. |
| -func runSystemUpdate(args []string) { log.Fatal("carlos system update: not yet wired to the console") } |
| -func runSystemStatus(args []string) { log.Fatal("carlos system status: not yet wired to the console") } |
| +// ---- the console's answers --------------------------------------------- |
| + |
| +// systemPointerRow is one channel in GET /api/cli/system/status's "pointers". |
| +// Error is the console saying it FOUND a pointer and could not verify it — |
| +// distinct from a null pointer, which means the channel has never been |
| +// promoted. Rendering the two the same way would hide the only one of them |
| +// that is an emergency. |
| +type systemPointerRow struct { |
| + Version string `json:"version"` |
| + PromotedAt time.Time `json:"promoted_at"` |
| + Spread string `json:"spread"` |
| + Source string `json:"source"` |
| + History int `json:"history"` |
| + Error string `json:"error"` |
| +} |
| + |
| +// systemStateRow is a box's last self-update outcome, as the box published it |
| +// in its heartbeat (internal/status.SystemState). |
| +type systemStateRow struct { |
| + Channel string `json:"channel"` |
| + Version string `json:"version"` |
| + PromotedAt time.Time `json:"promoted_at"` |
| + Outcome string `json:"outcome"` |
| + Reason string `json:"reason"` |
| + At time.Time `json:"at"` |
| +} |
| + |
| +type systemBoxRow struct { |
| + Label string `json:"label"` |
| + Region string `json:"region"` |
| + Platform string `json:"platform"` |
| + Channel string `json:"channel"` |
| + Release string `json:"release"` |
| + Target string `json:"target"` |
| + System *systemStateRow `json:"system"` |
| + Stale bool `json:"stale"` |
| +} |
| + |
| +// systemStatusResponse is the whole picture in one read. Partial says the box |
| +// list is INCOMPLETE — a fleet's heartbeats could not be read — so a short |
| +// list is never mistaken for a small fleet. |
| +type systemStatusResponse struct { |
| + Pointers map[string]*systemPointerRow `json:"pointers"` |
| + Boxes []systemBoxRow `json:"boxes"` |
| + Partial bool `json:"partial"` |
| +} |
| + |
| +// ---- carlos system update ---------------------------------------------- |
| + |
| +func runSystemUpdate(args []string) { |
| + fs := flag.NewFlagSet("system update", flag.ExitOnError) |
| + consoleFlag := fs.String("console", "", "console origin (default: the only logged-in one)") |
| + channel := fs.String("channel", "stable", "canary or stable") |
| + version := fs.String("version", "", "release tag to promote (default: the latest public release); with --file, the version the binary reports") |
| + file := fs.String("file", "", "a locally built binary instead of a public release (dev and pre-release)") |
| + platform := fs.String("platform", "linux-arm64", "with --file: the binary's <goos>-<goarch>") |
| + spread := fs.String("spread", "5m", "boxes jitter their restarts over this; 0 for a hotfix") |
| + yes := fs.Bool("yes", false, "do not ask") |
| + fs.Usage = flagUsage(fs, "") |
| + fs.Parse(args) |
| + if *file != "" && *version == "" { |
| + log.Fatal("--file needs --version: the console cannot run a foreign-arch binary to ask it") |
| + } |
| + sess := requireDeployAPISession(*consoleFlag) |
| + ctx := context.Background() |
| + |
| + // What this reaches, before it is done. |
| + st, body, err := sess.getJSON(ctx, "/api/cli/system/status") |
| + if err != nil { |
| + log.Fatalf("carlos system update: %v", err) |
| + } |
| + if st != http.StatusOK { |
| + log.Fatalf("carlos system update: %s", apiErrorFromBody(st, body)) |
| + } |
| + var cur systemStatusResponse |
| + _ = json.Unmarshal(body, &cur) |
| + var reach []string |
| + for _, b := range cur.Boxes { |
| + if b.Channel == *channel { |
| + reach = append(reach, fmt.Sprintf("%s (%s, running %s)", b.Label, orDash(b.Platform), orDash(b.Release))) |
| + } |
| + } |
| + what := "the latest public release" |
| + if *version != "" { |
| + what = *version |
| + } |
| + fmt.Fprintf(stdout, "promote %s to %s/%s — %d box(es) will roll over %s:\n", what, systemComponentRouter, *channel, len(reach), *spread) |
| + for _, r := range reach { |
| + fmt.Fprintf(stdout, " %s\n", r) |
| + } |
| + if len(reach) == 0 { |
| + mutedf(stdout, " (no box reports channel %s yet; boxes on an older build count as stable)\n", *channel) |
| + } |
| + if cur.Partial { |
| + fmt.Fprintf(stdout, " %s\n", colorFor(stdout).warn("some fleets could not be read; more boxes than these may roll")) |
| + } |
| + if !*yes { |
| + if !interactiveTerminal() { |
| + log.Fatal("no terminal to confirm on; pass --yes") |
| + } |
| + if !promptYes(stdout, "go ahead? [y/N] ") { |
| + return |
| + } |
| + } |
| + |
| + in := map[string]any{"channel": *channel, "spread": *spread} |
| + if *version != "" { |
| + in["version"] = *version |
| + } |
| + if *file != "" { |
| + handle, err := sess.uploadSystemBinary(ctx, *file, *version, *platform) |
| + if err != nil { |
| + log.Fatalf("upload: %v", err) |
| + } |
| + in["upload"] = handle |
| + } |
| + st, body, err = sess.postJSON(ctx, "/api/cli/system/update", in) |
| + if err != nil { |
| + log.Fatalf("carlos system update: %v", err) |
| + } |
| + if st != http.StatusOK { |
| + // 409 is the one refusal that carries somewhere to go: the id of the |
| + // promote already running on this channel, which the operator can |
| + // watch instead of starting a second one. |
| + var busy struct { |
| + Error string `json:"error"` |
| + Job string `json:"job"` |
| + } |
| + if st == http.StatusConflict && json.Unmarshal(body, &busy) == nil && busy.Job != "" { |
| + log.Fatalf("carlos system update: %s (job %s)", busy.Error, busy.Job) |
| + } |
| + log.Fatalf("carlos system update: %s", apiErrorFromBody(st, body)) |
| + } |
| + var started struct { |
| + Job string `json:"job"` |
| + } |
| + _ = json.Unmarshal(body, &started) |
| + if err := sess.watchSystemJob(ctx, started.Job, stdout); err != nil { |
| + log.Fatal(err) |
| + } |
| + fmt.Fprintf(stdout, "%s boxes on %s roll themselves; watch with: carlos system status --channel %s\n", |
| + colorFor(stdout).ok("promoted"), *channel, *channel) |
| +} |
| + |
| +// watchSystemJob polls the job and prints each step once, as deploy's |
| +// watchVersion does for a route. The console runs the promote off the |
| +// request precisely so a dropped connection cannot abandon it, so this is a |
| +// view of work already under way, not the work itself. |
| +func (s *apiSession) watchSystemJob(ctx context.Context, id string, w io.Writer) error { |
| + printed := 0 |
| + deadline := time.Now().Add(15 * time.Minute) |
| + for { |
| + st, body, err := s.getJSON(ctx, "/api/cli/system/jobs/"+id) |
| + if err != nil { |
| + return fmt.Errorf("job %s: %v", id, err) |
| + } |
| + if st != http.StatusOK { |
| + return fmt.Errorf("job %s: %s", id, apiErrorFromBody(st, body)) |
| + } |
| + var j struct { |
| + Steps []string `json:"steps"` |
| + Done bool `json:"done"` |
| + Error string `json:"error"` |
| + } |
| + _ = json.Unmarshal(body, &j) |
| + for ; printed < len(j.Steps); printed++ { |
| + fmt.Fprintf(w, " %s\n", j.Steps[printed]) |
| + } |
| + if j.Done { |
| + if j.Error != "" { |
| + return fmt.Errorf("carlos system update: %s", j.Error) |
| + } |
| + return nil |
| + } |
| + if time.Now().After(deadline) { |
| + return errors.New("carlos system update: the console's job did not finish in 15 minutes") |
| + } |
| + time.Sleep(time.Second) |
| + } |
| +} |
| + |
| +// uploadSystemBinary parks a locally built binary on the console and hands |
| +// back the handle a promote names. The meta part goes FIRST so the console |
| +// can refuse a bad version or platform before buffering the bytes. |
| +func (s *apiSession) uploadSystemBinary(ctx context.Context, path, version, platform string) (string, error) { |
| + st, body, err := s.c.doMultipartMethod(ctx, http.MethodPut, "/api/cli/system/upload", func(mw *multipart.Writer) error { |
| + meta, err := mw.CreateFormField("meta") |
| + if err != nil { |
| + return err |
| + } |
| + if err := json.NewEncoder(meta).Encode(map[string]string{"version": version, "platform": platform}); err != nil { |
| + return err |
| + } |
| + return writeFilePart(mw, "file", path) |
| + }) |
| + if err != nil { |
| + return "", err |
| + } |
| + if st != http.StatusOK { |
| + return "", apiErrorFromBody(st, body) |
| + } |
| + var out struct { |
| + Upload string `json:"upload"` |
| + } |
| + if err := json.Unmarshal(body, &out); err != nil || out.Upload == "" { |
| + return "", errors.New("the console accepted the upload but named no handle") |
| + } |
| + return out.Upload, nil |
| +} |
| + |
| +// ---- carlos system status ---------------------------------------------- |
| + |
| +func runSystemStatus(args []string) { |
| + fs := flag.NewFlagSet("system status", flag.ExitOnError) |
| + consoleFlag := fs.String("console", "", "console origin (default: the only logged-in one)") |
| + channel := fs.String("channel", "", "only boxes on this channel") |
| + fs.Usage = flagUsage(fs, "") |
| + fs.Parse(args) |
| + sess := requireDeployAPISession(*consoleFlag) |
| + st, body, err := sess.getJSON(context.Background(), "/api/cli/system/status") |
| + if err != nil { |
| + log.Fatalf("carlos system status: %v", err) |
| + } |
| + if st != http.StatusOK { |
| + log.Fatalf("carlos system status: %s", apiErrorFromBody(st, body)) |
| + } |
| + var resp systemStatusResponse |
| + if err := json.Unmarshal(body, &resp); err != nil { |
| + log.Fatalf("carlos system status: unreadable response: %v", err) |
| + } |
| + printSystemStatus(stdout, resp, *channel) |
| +} |
| + |
| +// printSystemStatus renders the pointers, then every box against them. It is |
| +// the whole verb minus the fetch, so the rendering is testable without a |
| +// console. |
| +func printSystemStatus(w io.Writer, resp systemStatusResponse, channel string) { |
| + c := colorFor(w) |
| + for _, ch := range []string{"canary", "stable"} { |
| + if channel != "" && ch != channel { |
| + continue |
| + } |
| + p := resp.Pointers[ch] |
| + switch { |
| + case p == nil: |
| + fmt.Fprintf(w, "%-7s (no pointer)\n", ch) |
| + case p.Error != "": |
| + // Never "(no pointer)": the console found one and could not |
| + // verify it, which is the opposite of never promoted. |
| + fmt.Fprintf(w, "%-7s %s\n", ch, c.bad("(pointer does not verify: "+p.Error+")")) |
| + default: |
| + // Built from what is actually there: a console that omits a |
| + // field must not make this print year 1 or a bare "spread". |
| + parts := []string{fmt.Sprintf("%-7s %s", ch, p.Version)} |
| + if !p.PromotedAt.IsZero() { |
| + parts = append(parts, "promoted "+p.PromotedAt.Format(time.RFC3339)) |
| + } |
| + if p.Spread != "" { |
| + parts = append(parts, "spread "+p.Spread) |
| + } |
| + if p.Source != "" { |
| + parts = append(parts, p.Source) |
| + } |
| + fmt.Fprintln(w, strings.Join(parts, " ")) |
| + } |
| + } |
| + fmt.Fprintln(w) |
| + if resp.Partial { |
| + fmt.Fprintln(w, c.warn("fleet boxes could not all be read; rows may be missing")) |
| + } |
| + for _, b := range resp.Boxes { |
| + if channel != "" && b.Channel != channel { |
| + continue |
| + } |
| + state := "" |
| + switch { |
| + case b.Stale: |
| + state = c.bad("UNREACHABLE") |
| + case b.Platform == "": |
| + state = c.warn("pre-upgrade (no platform reported; counts as stable)") |
| + case b.Target == "": |
| + state = "no target" |
| + case b.Release == b.Target: |
| + state = c.ok("current") |
| + default: |
| + state = c.warn("behind → " + b.Target) |
| + } |
| + fmt.Fprintf(w, "%-20s %-12s %-7s %-10s %s", b.Label, orDash(b.Platform), orDash(b.Channel), orDash(b.Release), state) |
| + if b.System != nil && b.System.Outcome != "applied" && b.System.Outcome != "noted" { |
| + fmt.Fprintf(w, " %s %s: %s", c.bad(b.System.Outcome), b.System.Version, b.System.Reason) |
| + } |
| + fmt.Fprintln(w) |
| + } |
| +} |
| + |
| +// ---- carlos system rollback -------------------------------------------- |
| + |
| func runSystemRollback(args []string) { |
| - log.Fatal("carlos system rollback: not yet wired to the console") |
| + fs := flag.NewFlagSet("system rollback", flag.ExitOnError) |
| + consoleFlag := fs.String("console", "", "console origin (default: the only logged-in one)") |
| + channel := fs.String("channel", "", "canary or stable (required)") |
| + yes := fs.Bool("yes", false, "do not ask") |
| + fs.Usage = flagUsage(fs, "") |
| + fs.Parse(args) |
| + if *channel == "" { |
| + log.Fatal("--channel is required") |
| + } |
| + sess := requireDeployAPISession(*consoleFlag) |
| + if !*yes { |
| + if !interactiveTerminal() { |
| + log.Fatal("no terminal to confirm on; pass --yes") |
| + } |
| + if !promptYes(stdout, fmt.Sprintf("point %s back at the release before this one? every box on it rolls back [y/N] ", *channel)) { |
| + return |
| + } |
| + } |
| + st, body, err := sess.postJSON(context.Background(), "/api/cli/system/rollback", map[string]string{"channel": *channel}) |
| + if err != nil { |
| + log.Fatalf("carlos system rollback: %v", err) |
| + } |
| + if st != http.StatusOK { |
| + log.Fatalf("carlos system rollback: %s", apiErrorFromBody(st, body)) |
| + } |
| + var out struct { |
| + Version string `json:"version"` |
| + } |
| + _ = json.Unmarshal(body, &out) |
| + fmt.Fprintf(stdout, "%s %s → %s\n", colorFor(stdout).ok("rolled back"), *channel, out.Version) |
| } |
amadan