| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "flag" |
| 8 | "fmt" |
| 9 | "io" |
| 10 | "log" |
| 11 | "mime/multipart" |
| 12 | "net/http" |
| 13 | "os" |
| 14 | "strings" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/dbreads" |
| 18 | "github.com/carlosframework/platform/internal/host" |
| 19 | "github.com/carlosframework/platform/internal/hostenv" |
| 20 | ) |
| 21 | |
| 22 | // carlos system: what is not an app (spec §0, the console-as-an-app |
| 23 | // dividing line), and what is true of the DEPLOYMENT rather than of one app. |
| 24 | // |
| 25 | // Two families live here. The first is the platform's own binary: one |
| 26 | // component today, the router — the substrate binary. `carlos system router |
| 27 | // update` is the same verb scoped to it, so the command Paul named works and |
| 28 | // a future split needs no new verb. The second is deployment-wide settings an |
| 29 | // operator owns, which today is `antivirus`: whether object scanning is on at |
| 30 | // all, a question whose answer should not require logging into a box to read |
| 31 | // a unit file. |
| 32 | // |
| 33 | // They are one command because they answer the same shape of question — what |
| 34 | // is this deployment doing — and they keep separate verbs because the |
| 35 | // authority differs: the update family wants an ops account OWNER or an |
| 36 | // address on CARLOS_SYSTEM_OPERATORS (local builds the owner's alone), antivirus |
| 37 | // wants an address on CARLOS_STORE_OPERATORS. |
| 38 | |
| 39 | const systemComponentRouter = "router" |
| 40 | |
| 41 | func systemVerb(args []string) (verb string, rest []string, component string) { |
| 42 | component = systemComponentRouter |
| 43 | if len(args) > 0 && args[0] == systemComponentRouter { |
| 44 | args = args[1:] |
| 45 | } |
| 46 | if len(args) == 0 { |
| 47 | return "", nil, component |
| 48 | } |
| 49 | switch args[0] { |
| 50 | // antivirus is a FAMILY, not a leaf: its own sub-verbs come back |
| 51 | // untouched in rest for runSystemAntivirusVerb to read. |
| 52 | // layout is a family too: migrate|revert|status stay in rest for |
| 53 | // runSystemLayout. |
| 54 | case "claims", "files", "guard-arm", "guard-exec", "journal-migrate", "floor-allows-prev", "update", "status", "rollback", "cancel-activation", "apply", "antivirus", "layout", "geoip", "seeds-floor", "legacy-routes", "box", "group", "install-channel", "join", "host-env", "db-worker", "help", "-h", "--help": |
| 55 | return args[0], args[1:], component |
| 56 | } |
| 57 | return "", args, component |
| 58 | } |
| 59 | |
| 60 | func runSystem(args []string) { |
| 61 | verb, rest, _ := systemVerb(args) |
| 62 | switch verb { |
| 63 | case "claims": |
| 64 | host.RunSystemClaims(rest) |
| 65 | case "files": |
| 66 | host.RunSystemFiles(rest) |
| 67 | case "guard-arm": |
| 68 | host.RunSystemGuardArm(rest) |
| 69 | case "guard-exec": |
| 70 | host.RunSystemGuardExec(rest) |
| 71 | case "journal-migrate": |
| 72 | host.RunSystemJournalMigrate(rest) |
| 73 | case "floor-allows-prev": |
| 74 | // Root's floor gate for the system units; not for operators. |
| 75 | host.RunSystemFloorAllowsPrev(rest) |
| 76 | |
| 77 | case "apply": |
| 78 | host.RunSystemApply(rest) |
| 79 | case "update": |
| 80 | runSystemUpdate(rest) |
| 81 | case "status": |
| 82 | runSystemStatus(rest) |
| 83 | case "rollback": |
| 84 | runSystemRollback(rest) |
| 85 | case "cancel-activation": |
| 86 | runSystemCancelActivation(rest) |
| 87 | case "antivirus": |
| 88 | runSystemAntivirusVerb(rest) |
| 89 | case "layout": |
| 90 | runSystemLayout(rest) |
| 91 | case "geoip": |
| 92 | runSystemGeoipVerb(rest) |
| 93 | case "seeds-floor": |
| 94 | runSystemSeedsFloorVerb(rest) |
| 95 | case "legacy-routes": |
| 96 | runSystemLegacyRoutesVerb(rest) |
| 97 | case "box": |
| 98 | runSystemBoxVerb(rest) |
| 99 | case "group": |
| 100 | runSystemGroupVerb(rest) |
| 101 | case "install-channel": |
| 102 | // A joining box's own release (region HA spec 2 §5 step 3); root. |
| 103 | host.RunSystemInstallChannel(rest) |
| 104 | case "join": |
| 105 | // Root's half of joining, behind carlos-system-join.service. |
| 106 | host.RunSystemJoin(rest) |
| 107 | case "host-env": |
| 108 | // host.env from and into the deployment's parameters (§5 step 2). |
| 109 | hostenv.RunSystemHostEnv(rest) |
| 110 | case "db-worker": |
| 111 | // The db-read worker (db region forward spec §8.3): systemd starts |
| 112 | // it per connection to carlos-dbread.socket. Never run by hand. |
| 113 | dbreads.RunWorker(rest) |
| 114 | case "help", "-h", "--help": |
| 115 | systemUsage() |
| 116 | default: |
| 117 | if msg := systemUnknownVerbMessage(args, rest); msg != "" { |
| 118 | fmt.Fprintf(os.Stderr, "%s\n\n", msg) |
| 119 | } |
| 120 | systemUsage() |
| 121 | os.Exit(2) |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | // systemUnknownVerbMessage is the line above the usage when no verb matched. |
| 126 | // It reads REST, not args, because systemVerb has already eaten the optional |
| 127 | // component: `carlos system router console` is an unknown verb "console", |
| 128 | // never an unknown verb "router". "" means say nothing at all — bare |
| 129 | // `carlos system` is someone asking for the usage, not a typo to report. |
| 130 | func systemUnknownVerbMessage(args, rest []string) string { |
| 131 | switch { |
| 132 | case len(rest) > 0: |
| 133 | return fmt.Sprintf("system: unknown verb %q", rest[0]) |
| 134 | case len(args) > 0: |
| 135 | return "system: no verb given" |
| 136 | default: |
| 137 | return "" |
| 138 | } |
| 139 | } |
| 140 | |
| 141 | func systemUsage() { |
| 142 | fmt.Fprint(os.Stderr, `usage: carlos system <verb> [flags] |
| 143 | carlos system router <verb> [flags] |
| 144 | carlos system antivirus <status|enable|disable|rotate-webhook-key> [flags] |
| 145 | carlos system geoip <publish|status|off> [flags] |
| 146 | carlos system seeds-floor publish --min-release <release> |
| 147 | carlos system legacy-routes assign --name <name> --to <account>/<app> |
| 148 | carlos system box role <box> <default|legacy|platform> |
| 149 | carlos system box publish-record |
| 150 | carlos system group role <region> <default|legacy> |
| 151 | carlos system host-env <write|seed> --deployment <name> --region <code> [flags] |
| 152 | |
| 153 | The platform's own binary (ops account owners and CARLOS_SYSTEM_OPERATORS): |
| 154 | |
| 155 | update promote a platform release to a channel; every box on it rolls itself |
| 156 | status every box: platform, channel, running build, channel target, last apply outcome |
| 157 | rollback point a channel back at the release before this one |
| 158 | cancel-activation |
| 159 | release a channel's activation reservation that was never |
| 160 | published, so promote and rollback work again |
| 161 | apply (root oneshot behind carlos-system-update.service; not for operators) |
| 162 | layout migrate|revert|status: move every instance into its own directory |
| 163 | and namespace, or back; see carlos system layout help |
| 164 | seeds-floor publish --min-release <release> |
| 165 | the first release whose boxes can place a seed, such as v0.27.0 |
| 166 | legacy-routes assign --name <name> --to <account>/<app> |
| 167 | `+systemLegacyRoutesVerb+` |
| 168 | box role <box> <default|legacy|platform> |
| 169 | whether new instances may be placed on a box. A legacy or |
| 170 | platform box keeps the instances it has and takes no new ones |
| 171 | group role <region> <default|legacy> |
| 172 | the same for every box a region's group has enrolled, and for |
| 173 | every box it enrolls later |
| 174 | box publish-record |
| 175 | run on a box as the agent user, with systemd-run -p User=carlos: |
| 176 | publish its AWS-signed identity record to the deployment |
| 177 | bucket, once. It never replaces or deletes a record |
| 178 | |
| 179 | A box a region's group launches (region HA): |
| 180 | |
| 181 | host-env write build host.env from the deployment's parameters (root) |
| 182 | host-env seed copy a box's host.env into the deployment's parameters |
| 183 | install-channel install the release this box's channel names (root) |
| 184 | join (root oneshot behind carlos-system-join.service; not for operators) |
| 185 | |
| 186 | Deployment-wide settings (CARLOS_STORE_OPERATORS): |
| 187 | |
| 188 | antivirus status is object scanning wired, switched on, and answering? |
| 189 | Reports wiring and policy separately, because they fail |
| 190 | separately. |
| 191 | antivirus enable turn object scanning on for this deployment. Refused if |
| 192 | no engine is wired: a record claiming enabled with |
| 193 | nothing behind it would report health where no scan can |
| 194 | run. |
| 195 | antivirus disable turn it off. Takes effect on the next tick — no restart, |
| 196 | which is the point of the switch living in a record |
| 197 | rather than in the unit file. |
| 198 | antivirus rotate-webhook-key |
| 199 | deliver the current master-derived callback secret to |
| 200 | every scanning-enabled app, one bounded batch at a |
| 201 | time. Skips apps already on that key, restarts the |
| 202 | rest, and is safe to run again. |
| 203 | |
| 204 | Platform GeoIP (ops account owners and CARLOS_SYSTEM_OPERATORS): |
| 205 | |
| 206 | geoip publish <file.mmdb> publish a MaxMind-format database (DB-IP Lite or |
| 207 | GeoLite2) to every edge. -force skips the shrink check |
| 208 | geoip status the published database, and whether every box |
| 209 | strips forged headers and has it |
| 210 | geoip off stop: edges drop the database and send no geo headers |
| 211 | |
| 212 | Enabling antivirus scans nothing by itself: a member still requests scanning |
| 213 | for their app, and an operator still grants it per store. |
| 214 | |
| 215 | The platform is not an app: the console is updated with carlos deploy --app console. |
| 216 | `) |
| 217 | } |
| 218 | |
| 219 | // ---- the console's answers --------------------------------------------- |
| 220 | |
| 221 | // systemPointerRow is one channel in GET /api/cli/system/status's "pointers". |
| 222 | // Error is the console saying it FOUND a pointer and could not verify it — |
| 223 | // distinct from a null pointer, which means the channel has never been |
| 224 | // promoted. Rendering the two the same way would hide the only one of them |
| 225 | // that is an emergency. |
| 226 | type systemPointerRow struct { |
| 227 | Version string `json:"version"` |
| 228 | PromotedAt time.Time `json:"promoted_at"` |
| 229 | Spread string `json:"spread"` |
| 230 | Source string `json:"source"` |
| 231 | History int `json:"history"` |
| 232 | Error string `json:"error"` |
| 233 | } |
| 234 | |
| 235 | // systemStateRow is a box's last self-update outcome, as the box published it |
| 236 | // in its heartbeat (internal/status.SystemState). |
| 237 | type systemStateRow struct { |
| 238 | Channel string `json:"channel"` |
| 239 | Version string `json:"version"` |
| 240 | PromotedAt time.Time `json:"promoted_at"` |
| 241 | Outcome string `json:"outcome"` |
| 242 | Reason string `json:"reason"` |
| 243 | At time.Time `json:"at"` |
| 244 | } |
| 245 | |
| 246 | type systemBoxRow struct { |
| 247 | Label string `json:"label"` |
| 248 | Region string `json:"region"` |
| 249 | Platform string `json:"platform"` |
| 250 | Channel string `json:"channel"` |
| 251 | Release string `json:"release"` |
| 252 | Target string `json:"target"` |
| 253 | System *systemStateRow `json:"system"` |
| 254 | Stale bool `json:"stale"` |
| 255 | } |
| 256 | |
| 257 | // systemStatusResponse is the whole picture in one read. Partial says the box |
| 258 | // list is INCOMPLETE — a fleet's heartbeats could not be read — so a short |
| 259 | // list is never mistaken for a small fleet. |
| 260 | type systemStatusResponse struct { |
| 261 | Pointers map[string]*systemPointerRow `json:"pointers"` |
| 262 | Boxes []systemBoxRow `json:"boxes"` |
| 263 | Partial bool `json:"partial"` |
| 264 | Bootstrap *systemBootstrapRow `json:"bootstrap"` |
| 265 | } |
| 266 | |
| 267 | // systemBootstrapRow is bootstrap/carlos, the binary a new box boots, |
| 268 | // against stable's signed linux-arm64 artifact (issue #316). nil when there |
| 269 | // is nothing to compare, or the console predates the field. |
| 270 | type systemBootstrapRow struct { |
| 271 | Version string `json:"version"` // stable's version |
| 272 | Key string `json:"key"` // stable's mirrored linux-arm64 artifact |
| 273 | State string `json:"state"` // current, behind or unreadable |
| 274 | Error string `json:"error"` |
| 275 | } |
| 276 | |
| 277 | // ---- carlos system update ---------------------------------------------- |
| 278 | |
| 279 | func runSystemUpdate(args []string) { |
| 280 | fs := flag.NewFlagSet("system update", flag.ExitOnError) |
| 281 | consoleFlag := consoleOption(fs, "console origin (default: the only logged-in one)") |
| 282 | channel := fs.String("channel", "stable", "canary or stable") |
| 283 | version := fs.String("version", "", "release tag to promote (default: the latest public release); with --file, the version the binary reports") |
| 284 | file := fs.String("file", "", "a locally built binary instead of a public release (dev and pre-release; ops account owners only)") |
| 285 | platform := fs.String("platform", "linux-arm64", "with --file: the binary's <goos>-<goarch>") |
| 286 | spread := fs.String("spread", "5m", "boxes jitter their restarts over this; 0 for a hotfix") |
| 287 | yes := fs.Bool("yes", false, "do not ask") |
| 288 | fs.Usage = flagUsage(fs, "") |
| 289 | fs.Parse(args) |
| 290 | if *file != "" && *version == "" { |
| 291 | log.Fatal("--file needs --version: the console cannot run a foreign-arch binary to ask it") |
| 292 | } |
| 293 | sess := requireDeployAPISession(*consoleFlag) |
| 294 | ctx := context.Background() |
| 295 | |
| 296 | // What this reaches, before it is done. |
| 297 | st, body, err := sess.getJSON(ctx, "/api/cli/system/status") |
| 298 | if err != nil { |
| 299 | log.Fatalf("carlos system update: %v", err) |
| 300 | } |
| 301 | if st != http.StatusOK { |
| 302 | log.Fatalf("carlos system update: %s", apiErrorFromBody(st, body)) |
| 303 | } |
| 304 | var cur systemStatusResponse |
| 305 | _ = json.Unmarshal(body, &cur) |
| 306 | var reach []string |
| 307 | for _, b := range cur.Boxes { |
| 308 | if b.Channel == *channel { |
| 309 | reach = append(reach, fmt.Sprintf("%s (%s, running %s)", b.Label, orDash(b.Platform), orDash(b.Release))) |
| 310 | } |
| 311 | } |
| 312 | what := "the latest public release" |
| 313 | if *version != "" { |
| 314 | what = *version |
| 315 | } |
| 316 | fmt.Fprintf(stdout, "promote %s to %s/%s — %d box(es) will roll over %s:\n", what, systemComponentRouter, *channel, len(reach), *spread) |
| 317 | for _, r := range reach { |
| 318 | fmt.Fprintf(stdout, " %s\n", r) |
| 319 | } |
| 320 | if len(reach) == 0 { |
| 321 | mutedf(stdout, " (no box reports channel %s yet; boxes on an older build count as stable)\n", *channel) |
| 322 | } |
| 323 | if cur.Partial { |
| 324 | fmt.Fprintf(stdout, " %s\n", colorFor(stdout).warn("some fleets could not be read; more boxes than these may roll")) |
| 325 | } |
| 326 | if msg := systemLadderWarning(cur.Pointers, *channel, *version); msg != "" { |
| 327 | fmt.Fprintf(stdout, "%s\n", colorFor(stdout).warn(msg)) |
| 328 | } |
| 329 | if !*yes { |
| 330 | if !interactiveTerminal() { |
| 331 | log.Fatal("no terminal to confirm on; pass --yes") |
| 332 | } |
| 333 | if !promptYes(stdout, "go ahead? [y/N] ") { |
| 334 | return |
| 335 | } |
| 336 | } |
| 337 | |
| 338 | in := map[string]any{"channel": *channel, "spread": *spread} |
| 339 | if *version != "" { |
| 340 | in["version"] = *version |
| 341 | } |
| 342 | if *file != "" { |
| 343 | handle, err := sess.uploadSystemBinary(ctx, *file, *version, *platform) |
| 344 | if err != nil { |
| 345 | log.Fatalf("upload: %v", err) |
| 346 | } |
| 347 | in["upload"] = handle |
| 348 | } |
| 349 | st, body, err = sess.postJSON(ctx, "/api/cli/system/update", in) |
| 350 | if err != nil { |
| 351 | log.Fatalf("carlos system update: %v", err) |
| 352 | } |
| 353 | if st != http.StatusOK { |
| 354 | // 409 is the one refusal that carries somewhere to go: the id of the |
| 355 | // promote already running on this channel, which the operator can |
| 356 | // watch instead of starting a second one. |
| 357 | var busy struct { |
| 358 | Error string `json:"error"` |
| 359 | Job string `json:"job"` |
| 360 | } |
| 361 | if st == http.StatusConflict && json.Unmarshal(body, &busy) == nil && busy.Job != "" { |
| 362 | log.Fatalf("carlos system update: %s (job %s)", busy.Error, busy.Job) |
| 363 | } |
| 364 | log.Fatalf("carlos system update: %s", apiErrorFromBody(st, body)) |
| 365 | } |
| 366 | var started struct { |
| 367 | Job string `json:"job"` |
| 368 | } |
| 369 | _ = json.Unmarshal(body, &started) |
| 370 | if err := sess.watchSystemJob(ctx, started.Job, stdout); err != nil { |
| 371 | log.Fatal(err) |
| 372 | } |
| 373 | fmt.Fprintf(stdout, "%s boxes on %s roll themselves; watch with: carlos system status --channel %s\n", |
| 374 | colorFor(stdout).ok("promoted"), *channel, *channel) |
| 375 | } |
| 376 | |
| 377 | // systemLadderWarning is the canary → stable ladder, said out loud and not |
| 378 | // enforced. A promote straight to stable is a legitimate thing to do — a |
| 379 | // hotfix at three in the morning, a build that has already proved itself |
| 380 | // somewhere else — so this is a line to read, never a refusal: the operator |
| 381 | // is the one who knows whether the rung was skipped on purpose. |
| 382 | // |
| 383 | // It stays quiet unless it is certain. Without --version the console is |
| 384 | // promoting "the latest public release", a tag only the server resolves, so |
| 385 | // there is nothing here to compare against the canary pointer and a warning |
| 386 | // would be a guess. Only stable has a rung below it worth naming. |
| 387 | func systemLadderWarning(pointers map[string]*systemPointerRow, channel, version string) string { |
| 388 | if channel != "stable" || version == "" { |
| 389 | return "" |
| 390 | } |
| 391 | if canary := pointers["canary"]; canary != nil && canary.Version == version { |
| 392 | return "" |
| 393 | } |
| 394 | return fmt.Sprintf("warning: %s has not been on canary; promoting straight to stable", version) |
| 395 | } |
| 396 | |
| 397 | // watchSystemJob polls the job and prints each step once, as deploy's |
| 398 | // watchVersion does for a route. The console runs the promote off the |
| 399 | // request precisely so a dropped connection cannot abandon it, so this is a |
| 400 | // view of work already under way, not the work itself. |
| 401 | func (s *apiSession) watchSystemJob(ctx context.Context, id string, w io.Writer) error { |
| 402 | printed := 0 |
| 403 | deadline := time.Now().Add(15 * time.Minute) |
| 404 | for { |
| 405 | st, body, err := s.getJSON(ctx, "/api/cli/system/jobs/"+id) |
| 406 | if err != nil { |
| 407 | return fmt.Errorf("job %s: %v", id, err) |
| 408 | } |
| 409 | if st != http.StatusOK { |
| 410 | return fmt.Errorf("job %s: %s", id, apiErrorFromBody(st, body)) |
| 411 | } |
| 412 | var j struct { |
| 413 | Steps []string `json:"steps"` |
| 414 | Done bool `json:"done"` |
| 415 | Error string `json:"error"` |
| 416 | } |
| 417 | _ = json.Unmarshal(body, &j) |
| 418 | for ; printed < len(j.Steps); printed++ { |
| 419 | fmt.Fprintf(w, " %s\n", j.Steps[printed]) |
| 420 | } |
| 421 | if j.Done { |
| 422 | if j.Error != "" { |
| 423 | return fmt.Errorf("carlos system update: %s", j.Error) |
| 424 | } |
| 425 | return nil |
| 426 | } |
| 427 | if time.Now().After(deadline) { |
| 428 | return errors.New("carlos system update: the console's job did not finish in 15 minutes") |
| 429 | } |
| 430 | time.Sleep(time.Second) |
| 431 | } |
| 432 | } |
| 433 | |
| 434 | // uploadSystemBinary parks a locally built binary on the console and hands |
| 435 | // back the handle a promote names. The meta part goes FIRST so the console |
| 436 | // can refuse a bad version or platform before buffering the bytes. |
| 437 | func (s *apiSession) uploadSystemBinary(ctx context.Context, path, version, platform string) (string, error) { |
| 438 | st, body, err := s.c.doMultipartMethod(ctx, http.MethodPut, "/api/cli/system/upload", func(mw *multipart.Writer) error { |
| 439 | meta, err := mw.CreateFormField("meta") |
| 440 | if err != nil { |
| 441 | return err |
| 442 | } |
| 443 | if err := json.NewEncoder(meta).Encode(map[string]string{"version": version, "platform": platform}); err != nil { |
| 444 | return err |
| 445 | } |
| 446 | return writeFilePart(mw, "file", path) |
| 447 | }) |
| 448 | if err != nil { |
| 449 | return "", err |
| 450 | } |
| 451 | if st != http.StatusOK { |
| 452 | return "", apiErrorFromBody(st, body) |
| 453 | } |
| 454 | var out struct { |
| 455 | Upload string `json:"upload"` |
| 456 | } |
| 457 | if err := json.Unmarshal(body, &out); err != nil || out.Upload == "" { |
| 458 | return "", errors.New("the console accepted the upload but named no handle") |
| 459 | } |
| 460 | return out.Upload, nil |
| 461 | } |
| 462 | |
| 463 | // ---- carlos system status ---------------------------------------------- |
| 464 | |
| 465 | func runSystemStatus(args []string) { |
| 466 | fs := flag.NewFlagSet("system status", flag.ExitOnError) |
| 467 | consoleFlag := consoleOption(fs, "console origin (default: the only logged-in one)") |
| 468 | channel := fs.String("channel", "", "only boxes on this channel") |
| 469 | fs.Usage = flagUsage(fs, "") |
| 470 | fs.Parse(args) |
| 471 | sess := requireDeployAPISession(*consoleFlag) |
| 472 | st, body, err := sess.getJSON(context.Background(), "/api/cli/system/status") |
| 473 | if err != nil { |
| 474 | log.Fatalf("carlos system status: %v", err) |
| 475 | } |
| 476 | if st != http.StatusOK { |
| 477 | log.Fatalf("carlos system status: %s", apiErrorFromBody(st, body)) |
| 478 | } |
| 479 | var resp systemStatusResponse |
| 480 | if err := json.Unmarshal(body, &resp); err != nil { |
| 481 | log.Fatalf("carlos system status: unreadable response: %v", err) |
| 482 | } |
| 483 | printSystemStatus(stdout, resp, *channel) |
| 484 | } |
| 485 | |
| 486 | // printSystemStatus renders the pointers, then every box against them. It is |
| 487 | // the whole verb minus the fetch, so the rendering is testable without a |
| 488 | // console. |
| 489 | func printSystemStatus(w io.Writer, resp systemStatusResponse, channel string) { |
| 490 | c := colorFor(w) |
| 491 | for _, ch := range []string{"canary", "stable"} { |
| 492 | if channel != "" && ch != channel { |
| 493 | continue |
| 494 | } |
| 495 | p := resp.Pointers[ch] |
| 496 | switch { |
| 497 | case p == nil: |
| 498 | fmt.Fprintf(w, "%-7s (no pointer)\n", ch) |
| 499 | case p.Error != "": |
| 500 | // Never "(no pointer)": the console found one and could not |
| 501 | // verify it, which is the opposite of never promoted. |
| 502 | fmt.Fprintf(w, "%-7s %s\n", ch, c.bad("(pointer does not verify: "+p.Error+")")) |
| 503 | default: |
| 504 | // Built from what is actually there: a console that omits a |
| 505 | // field must not make this print year 1 or a bare "spread". |
| 506 | parts := []string{fmt.Sprintf("%-7s %s", ch, p.Version)} |
| 507 | if !p.PromotedAt.IsZero() { |
| 508 | parts = append(parts, "promoted "+p.PromotedAt.Format(time.RFC3339)) |
| 509 | } |
| 510 | if p.Spread != "" { |
| 511 | parts = append(parts, "spread "+p.Spread) |
| 512 | } |
| 513 | if p.Source != "" { |
| 514 | parts = append(parts, p.Source) |
| 515 | } |
| 516 | fmt.Fprintln(w, strings.Join(parts, " ")) |
| 517 | } |
| 518 | } |
| 519 | // What a new box would boot (issue #316): quiet unless it is not stable's |
| 520 | // build, and only where stable is shown. |
| 521 | if b := resp.Bootstrap; b != nil && (channel == "" || channel == "stable") { |
| 522 | switch b.State { |
| 523 | case "behind": |
| 524 | fmt.Fprintln(w, c.warn(fmt.Sprintf("bootstrap/carlos is not stable's build (%s), so a new box would boot a different one. Copy %s over it.", b.Version, b.Key))) |
| 525 | case "unreadable": |
| 526 | fmt.Fprintln(w, c.warn("bootstrap/carlos could not be read, so this can't say what a new box would boot: "+b.Error)) |
| 527 | } |
| 528 | } |
| 529 | fmt.Fprintln(w) |
| 530 | if resp.Partial { |
| 531 | fmt.Fprintln(w, c.warn("fleet boxes could not all be read; rows may be missing")) |
| 532 | } |
| 533 | for _, b := range resp.Boxes { |
| 534 | if channel != "" && b.Channel != channel { |
| 535 | continue |
| 536 | } |
| 537 | state := "" |
| 538 | switch { |
| 539 | case b.Stale: |
| 540 | state = c.bad("UNREACHABLE") |
| 541 | case b.Platform == "": |
| 542 | state = c.warn("pre-upgrade (no platform reported; counts as stable)") |
| 543 | case b.Target == "": |
| 544 | state = "no target" |
| 545 | case b.Release == b.Target: |
| 546 | state = c.ok("current") |
| 547 | default: |
| 548 | state = c.warn("behind → " + b.Target) |
| 549 | } |
| 550 | fmt.Fprintf(w, "%-20s %-12s %-7s %-10s %s", b.Label, orDash(b.Platform), orDash(b.Channel), orDash(b.Release), state) |
| 551 | if b.System != nil && b.System.Outcome != "applied" && b.System.Outcome != "noted" { |
| 552 | fmt.Fprintf(w, " %s %s: %s", c.bad(b.System.Outcome), b.System.Version, b.System.Reason) |
| 553 | } |
| 554 | fmt.Fprintln(w) |
| 555 | } |
| 556 | } |
| 557 | |
| 558 | // ---- carlos system rollback -------------------------------------------- |
| 559 | |
| 560 | func runSystemRollback(args []string) { |
| 561 | fs := flag.NewFlagSet("system rollback", flag.ExitOnError) |
| 562 | consoleFlag := consoleOption(fs, "console origin (default: the only logged-in one)") |
| 563 | channel := fs.String("channel", "", "canary or stable (required)") |
| 564 | yes := fs.Bool("yes", false, "do not ask") |
| 565 | fs.Usage = flagUsage(fs, "") |
| 566 | fs.Parse(args) |
| 567 | if *channel == "" { |
| 568 | log.Fatal("--channel is required") |
| 569 | } |
| 570 | sess := requireDeployAPISession(*consoleFlag) |
| 571 | if !*yes { |
| 572 | if !interactiveTerminal() { |
| 573 | log.Fatal("no terminal to confirm on; pass --yes") |
| 574 | } |
| 575 | if !promptYes(stdout, fmt.Sprintf("point %s back at the release before this one? every box on it rolls back [y/N] ", *channel)) { |
| 576 | return |
| 577 | } |
| 578 | } |
| 579 | st, body, err := sess.postJSON(context.Background(), "/api/cli/system/rollback", map[string]string{"channel": *channel}) |
| 580 | if err != nil { |
| 581 | log.Fatalf("carlos system rollback: %v", err) |
| 582 | } |
| 583 | if st != http.StatusOK { |
| 584 | log.Fatalf("carlos system rollback: %s", apiErrorFromBody(st, body)) |
| 585 | } |
| 586 | var out struct { |
| 587 | Version string `json:"version"` |
| 588 | } |
| 589 | _ = json.Unmarshal(body, &out) |
| 590 | fmt.Fprintf(stdout, "%s %s → %s\n", colorFor(stdout).ok("rolled back"), *channel, out.Version) |
| 591 | } |
| 592 | |
| 593 | // ---- carlos system antivirus ------------------------------------------- |
| 594 | |
| 595 | // runSystemAntivirusVerb reads the sub-verb off rest. systemVerb hands the |
| 596 | // family through whole, so the shape of `carlos system antivirus <verb>` is |
| 597 | // decided in one place rather than smeared across the dispatch. |
| 598 | func runSystemAntivirusVerb(rest []string) { |
| 599 | if len(rest) == 0 { |
| 600 | fmt.Fprint(os.Stderr, "system antivirus: no verb given\n\n") |
| 601 | systemUsage() |
| 602 | os.Exit(2) |
| 603 | } |
| 604 | switch rest[0] { |
| 605 | case "status", "enable", "disable", "rotate-webhook-key": |
| 606 | runSystemAntivirus(rest[1:], rest[0]) |
| 607 | default: |
| 608 | fmt.Fprintf(os.Stderr, "system antivirus: unknown verb %q\n\n", rest[0]) |
| 609 | systemUsage() |
| 610 | os.Exit(2) |
| 611 | } |
| 612 | } |
| 613 | |
| 614 | func runSystemAntivirus(args []string, verb string) { |
| 615 | fs := flag.NewFlagSet("system antivirus "+verb, flag.ExitOnError) |
| 616 | consoleFlag := consoleOption(fs, "console to act through (default: the only logged-in one)") |
| 617 | fs.Usage = flagUsage(fs, "") |
| 618 | fs.Parse(args) |
| 619 | |
| 620 | sess := requireStoreAPISession(*consoleFlag) |
| 621 | printUsing(stdout, sess) |
| 622 | |
| 623 | ctx := context.Background() |
| 624 | var res apiSystemAntivirusReply |
| 625 | var err error |
| 626 | switch verb { |
| 627 | case "status": |
| 628 | res, err = sess.systemAntivirusStatus(ctx) |
| 629 | case "enable": |
| 630 | res, err = sess.systemAntivirusSet(ctx, true) |
| 631 | case "disable": |
| 632 | res, err = sess.systemAntivirusSet(ctx, false) |
| 633 | case "rotate-webhook-key": |
| 634 | warnf(stdout, "this rotation restarts every scanning-enabled app whose callback secret changes") |
| 635 | for { |
| 636 | rotated, rotateErr := sess.systemWebhookRotate(ctx) |
| 637 | if rotateErr != nil { |
| 638 | log.Fatalf("system antivirus %s: %v", verb, rotateErr) |
| 639 | } |
| 640 | fmt.Fprintf(stdout, "rotated callback secrets for %d apps; %d remain\n", rotated.Rotated, rotated.Remaining) |
| 641 | if rotated.Remaining == 0 { |
| 642 | break |
| 643 | } |
| 644 | if rotated.Rotated == 0 { |
| 645 | time.Sleep(2 * time.Second) |
| 646 | } |
| 647 | } |
| 648 | return |
| 649 | } |
| 650 | if err != nil { |
| 651 | log.Fatalf("system antivirus %s: %v", verb, err) |
| 652 | } |
| 653 | printSystemAntivirus(stdout, res) |
| 654 | } |
| 655 | |
| 656 | // printSystemAntivirus reports wiring and policy as the two separate facts |
| 657 | // they are. An operator debugging "why is nothing scanning" needs to know |
| 658 | // which half is missing, and a single "healthy" line would send half of them |
| 659 | // to the wrong place. |
| 660 | func printSystemAntivirus(w io.Writer, res apiSystemAntivirusReply) { |
| 661 | c := colorFor(w) |
| 662 | fmt.Fprintf(w, "%s\n", c.head("object scanning, deployment-wide")) |
| 663 | fmt.Fprintf(w, "Apps with queued scans: %d\n", res.QueueAppsWithWork) |
| 664 | |
| 665 | if !res.Wired { |
| 666 | warnf(w, "no engine is wired on this deployment — an operator sets CARLOS_CLAMD_ADDR") |
| 667 | } else if res.EngineError != "" { |
| 668 | warnf(w, "the engine is wired but not answering: %s", res.EngineError) |
| 669 | } else { |
| 670 | fmt.Fprintf(w, " engine: %s\n", res.EngineWired) |
| 671 | if res.Signature != "" { |
| 672 | fmt.Fprintf(w, " engine database now: %s\n", res.Signature) |
| 673 | if age, ok := signatureAge(res.Signature, time.Now()); ok { |
| 674 | max := time.Duration(res.MaxSignatureAgeSeconds) * time.Second |
| 675 | if max > 0 && age > max { |
| 676 | warnf(w, "those definitions are %d days old — update the engine's database", |
| 677 | int(age.Hours()/24)) |
| 678 | } |
| 679 | } |
| 680 | } |
| 681 | } |
| 682 | |
| 683 | if res.RecordError != "" { |
| 684 | warnf(w, "the scanning record could not be read: %s", res.RecordError) |
| 685 | return |
| 686 | } |
| 687 | |
| 688 | if !res.Enabled { |
| 689 | mutedf(w, " scanning is OFF for this deployment — carlos system antivirus enable") |
| 690 | if res.By != "" { |
| 691 | mutedf(w, " turned off by %s at %s", res.By, res.At) |
| 692 | } |
| 693 | return |
| 694 | } |
| 695 | fmt.Fprintf(w, " scanning is ON, enabled by %s at %s\n", res.By, res.At) |
| 696 | mutedf(w, " this alone scans nothing: a member requests scanning for their app,") |
| 697 | mutedf(w, " and an operator grants it per store (carlos store scan grant)") |
| 698 | } |
| 699 | |
| 700 | // ---- carlos system layout ---------------------------------------------- |
| 701 | |
| 702 | // runSystemLayout is `carlos system layout migrate|revert|status` |
| 703 | // (isolated-exec design §8.2). migrate and revert record an intent through |
| 704 | // the console's layout door and return at once: the boxes in scope |
| 705 | // converge on their next serial move, and `carlos status --layout` is the |
| 706 | // view of that. --force maps to the intent's Force field: an awake |
| 707 | // instance is stopped for its move now instead of waiting out its grace. |
| 708 | func runSystemLayout(args []string) { |
| 709 | if len(args) == 0 { |
| 710 | fmt.Fprint(os.Stderr, systemLayoutUsage) |
| 711 | os.Exit(2) |
| 712 | } |
| 713 | sub, rest := args[0], args[1:] |
| 714 | switch sub { |
| 715 | case "help", "-h", "--help": |
| 716 | fmt.Fprint(os.Stderr, systemLayoutUsage) |
| 717 | return |
| 718 | case "status", "migrate", "revert": |
| 719 | default: |
| 720 | fmt.Fprintf(os.Stderr, "system layout: unknown verb %q\n\n%s", sub, systemLayoutUsage) |
| 721 | os.Exit(2) |
| 722 | } |
| 723 | fs := flag.NewFlagSet("system layout "+sub, flag.ExitOnError) |
| 724 | consoleFlag := consoleOption(fs, "console origin (default: the only logged-in one)") |
| 725 | box := fs.String("box", "", "one box's label; the default is every box") |
| 726 | force := fs.Bool("force", false, "stop awake instances for the move now instead of waiting for them to sleep (migrate and revert)") |
| 727 | yes := fs.Bool("yes", false, "do not ask") |
| 728 | fs.Usage = flagUsage(fs, systemLayoutUsage) |
| 729 | fs.Parse(rest) |
| 730 | sess := requireDeployAPISession(*consoleFlag) |
| 731 | ctx := context.Background() |
| 732 | |
| 733 | if sub == "status" { |
| 734 | st, body, err := sess.getJSON(ctx, "/api/cli/system/layout") |
| 735 | if err != nil { |
| 736 | log.Fatalf("carlos system layout status: %v", err) |
| 737 | } |
| 738 | if st != http.StatusOK { |
| 739 | log.Fatalf("carlos system layout status: %s", systemLayoutDoorError(st, body)) |
| 740 | } |
| 741 | var out systemLayoutResponse |
| 742 | if err := json.Unmarshal(body, &out); err != nil { |
| 743 | log.Fatalf("carlos system layout status: unreadable response: %v", err) |
| 744 | } |
| 745 | printSystemLayoutStatus(stdout, out) |
| 746 | return |
| 747 | } |
| 748 | |
| 749 | mode, what := "nested", "move every instance on %s into its own directory and namespace" |
| 750 | if sub == "revert" { |
| 751 | mode, what = "flat", "move every isolated instance on %s back to the shared directory" |
| 752 | } |
| 753 | scope := "every box" |
| 754 | if *box != "" { |
| 755 | scope = "box " + *box |
| 756 | } |
| 757 | if !*yes { |
| 758 | if !interactiveTerminal() { |
| 759 | log.Fatal("no terminal to confirm on; pass --yes") |
| 760 | } |
| 761 | how := "each one is stopped and moved while it is asleep, or after an hour awake" |
| 762 | if *force { |
| 763 | how = "awake instances are stopped for the move at once" |
| 764 | } |
| 765 | if !promptYes(stdout, fmt.Sprintf(what+"? %s [y/N] ", scope, how)) { |
| 766 | return |
| 767 | } |
| 768 | } |
| 769 | st, body, err := sess.postJSON(ctx, "/api/cli/system/layout", map[string]any{"mode": mode, "box": *box, "force": *force}) |
| 770 | if err != nil { |
| 771 | log.Fatalf("carlos system layout %s: %v", sub, err) |
| 772 | } |
| 773 | if st != http.StatusOK { |
| 774 | log.Fatalf("carlos system layout %s: %s", sub, systemLayoutDoorError(st, body)) |
| 775 | } |
| 776 | fmt.Fprintf(stdout, "%s layout %s for %s; boxes converge on their next serial move. Watch with: carlos status --layout\n", |
| 777 | colorFor(stdout).ok("recorded"), mode, scope) |
| 778 | } |
| 779 | |
| 780 | // systemLayoutResponse is the layout door's answer (console |
| 781 | // apiSystemLayoutResponse). Mode "none" means nothing was ever recorded. |
| 782 | type systemLayoutResponse struct { |
| 783 | Mode string `json:"mode"` |
| 784 | Box string `json:"box"` |
| 785 | ID string `json:"id"` |
| 786 | RequestedAt string `json:"requested_at"` |
| 787 | Force bool `json:"force"` |
| 788 | } |
| 789 | |
| 790 | // systemLayoutDoorError is apiErrorFromBody with the one answer this door |
| 791 | // adds: a bare 404 from a console that predates the door, which the |
| 792 | // generic membership message would misdescribe. |
| 793 | func systemLayoutDoorError(st int, body []byte) error { |
| 794 | if st == http.StatusNotFound { |
| 795 | var e struct { |
| 796 | Error string `json:"error"` |
| 797 | } |
| 798 | if json.Unmarshal(body, &e) != nil || e.Error == "" { |
| 799 | return errors.New("not found. this needs an owner of the ops account or an address on CARLOS_SYSTEM_OPERATORS, " + |
| 800 | "or this console predates `carlos system layout` (deploy a console built from current main)") |
| 801 | } |
| 802 | } |
| 803 | return apiErrorFromBody(st, body) |
| 804 | } |
| 805 | |
| 806 | // printSystemLayoutStatus renders the recorded intent. It is the fleet's |
| 807 | // INTENT, not what the boxes have done: that is `carlos status --layout`, |
| 808 | // which the line points at. |
| 809 | func printSystemLayoutStatus(w io.Writer, out systemLayoutResponse) { |
| 810 | if out.Mode == "none" { |
| 811 | fmt.Fprintln(w, "no layout intent recorded; boxes use the nested layout by default once their isolation is ready (CARLOS_LAYOUT_DEFAULT=flat in a box's host.env opts it out)") |
| 812 | fmt.Fprintln(w, "to choose the layout yourself, record one: carlos system layout migrate [--box <label>] or carlos system layout revert") |
| 813 | fmt.Fprintln(w, "which boxes are nested, and which by default: carlos status --layout") |
| 814 | return |
| 815 | } |
| 816 | scope := "every box" |
| 817 | if out.Box != "" { |
| 818 | scope = "box " + out.Box |
| 819 | } |
| 820 | force := "" |
| 821 | if out.Force { |
| 822 | force = ", force" |
| 823 | } |
| 824 | fmt.Fprintf(w, "intent: %s for %s (recorded %s%s)\n", out.Mode, scope, out.RequestedAt, force) |
| 825 | fmt.Fprintln(w, "what each box has done: carlos status --layout") |
| 826 | } |
| 827 | |
| 828 | const systemLayoutUsage = `usage: carlos system layout migrate [--box <label>] [--force] [--yes] |
| 829 | carlos system layout revert [--box <label>] [--force] [--yes] |
| 830 | carlos system layout status |
| 831 | |
| 832 | migrate records the intent "nested": every instance the activator manages on |
| 833 | the boxes in scope is stopped while asleep, moved into its own directory, and |
| 834 | from its next wake runs in its own mount and pid namespace, where it can read |
| 835 | its own files and nothing else. revert records "flat" and moves them back; run |
| 836 | it, and wait for carlos status --layout to show every instance flat, before |
| 837 | rolling the platform back to a release from before the layout. status prints |
| 838 | the recorded intent. |
| 839 | |
| 840 | With no intent recorded, a box whose isolation is ready uses the nested |
| 841 | layout by default (CARLOS_LAYOUT_DEFAULT in its host.env; flat opts out). |
| 842 | revert is the fleet-wide opt-out; migrate --box is how to move one box at a |
| 843 | time instead. |
| 844 | |
| 845 | --box scopes the request to one box, so a fleet moves one box at a time. |
| 846 | --force stops an awake instance for its move at once instead of giving it an |
| 847 | hour to go to sleep first. Ops account owners and CARLOS_SYSTEM_OPERATORS. |
| 848 | ` |
| 849 | |
amadan