amadan
Merge branch 'route-replication-flag' into main
12 files changed,
+1109
−43
apimode.go+1 −0cli.go+180 −12docs/site/cli.md+36 −3internal/activator/activator.go+8 −3internal/console/apiroutes.go+7 −0internal/console/apiroutes_test.go+36 −1internal/provision/reconcile.go+45 −2internal/provision/reconcile_test.go+47 −0internal/registry/registry.go+167 −14internal/registry/replication_test.go+219 −0route_replication_test.go+333 −0routes_cli.go+30 −8
CI cancelled — superseded by a newer push — run details
diff --git a/apimode.go b/apimode.go| index 9650bccf4..1c7aebe66 100644 |
| --- a/apimode.go |
| +++ b/apimode.go |
| @@ -2517,6 +2517,7 @@ type apiRouteRow struct { |
| Unit string `json:"unit"` |
| Backing string `json:"backing"` |
| Provisioned bool `json:"provisioned"` |
| + Replication string `json:"replication"` |
| } |
| // apiRoutePathRow is one path route of the same answer: a leading URL |
diff --git a/cli.go b/cli.go| index 0652d5ac7..8b3d7eb88 100644 |
| --- a/cli.go |
| +++ b/cli.go |
| @@ -325,14 +325,23 @@ func runInstances(args []string) { |
| // app's default bundle, which is a real answer, so it prints blank |
| // rather than being singled out. Off-box, `carlos routes` answers the |
| // same question through the console. |
| - fmt.Fprintln(w, "HOST\tAPP\tKIND\tCHANNEL\tENV\tSTATUS\tVERSION\tHIB\tBACKING\tUNIT") |
| + // |
| + // REPL is the rung of the replication ladder this route's database is |
| + // replicated on (discussion #10). It prints beside HIB because the two |
| + // are one fact: the class is read only where the activator replicates, |
| + // which is a hibernating route. An empty cell means the pre-ladder |
| + // default — change-driven ~1s sync — and until `carlos route |
| + // --replication` existed it was the honest answer for every route on |
| + // every box, which is precisely why the column had to become visible at |
| + // the same moment it became settable. |
| + fmt.Fprintln(w, "HOST\tAPP\tKIND\tCHANNEL\tENV\tSTATUS\tVERSION\tHIB\tREPL\tBACKING\tUNIT") |
| for _, t := range routes { |
| hib := "" |
| if t.Hibernate { |
| hib = "yes" |
| } |
| - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", |
| - t.Host, t.App, t.Kind, t.Channel, t.Environment, t.Status, t.Version, hib, t.Backing, t.Unit) |
| + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", |
| + t.Host, t.App, t.Kind, t.Channel, t.Environment, t.Status, t.Version, hib, t.Replication, t.Backing, t.Unit) |
| } |
| w.Flush() |
| } |
| @@ -475,6 +484,8 @@ func runRoute(args []string) { |
| `systemd unit that owns this route's process (e.g. console.service), so adoption can cycle it; "-" clears the stamp (host only, no --app)`) |
| tunnel := fs.String("tunnel", "", |
| `name of the reverse channel backing this route at the edge; "-" clears to a Socket/Addr-routed upstream (host only, no --app)`) |
| + replication := fs.String("replication", "", |
| + `replication ladder rung for this route's database: `+strings.Join(registry.ReplicationClasses(), " | ")+`; "-" clears to the default (host only, no --app)`) |
| allowVersionChange := fs.Bool("allow-version-change", false, |
| "allow repointing to a channel that RESOLVES but holds a different version than the route currently serves "+ |
| "(never overrides an unresolvable target, a bad signature, or a missing pubkey)") |
| @@ -495,7 +506,11 @@ func runRoute(args []string) { |
| // "--unit -", the sentinel --environment and --tunnel already use, so that a |
| // bare `--unit ""` reads as the mistake it almost certainly is rather than |
| // as a silent unstamp. |
| - hibernateSet, backingSet, unitSet := false, false, false |
| + // --replication is tri-state on --unit's exact grounds: "" is this |
| + // column's default (the pre-ladder every-second sync), so a flag left |
| + // unmentioned and a flag set to "" cannot be told apart by value, and its |
| + // CLEAR spelling is the same "-" sentinel. |
| + hibernateSet, backingSet, unitSet, replicationSet := false, false, false, false |
| fs.Visit(func(f *flag.Flag) { |
| switch f.Name { |
| case "hibernate": |
| @@ -504,6 +519,8 @@ func runRoute(args []string) { |
| backingSet = true |
| case "unit": |
| unitSet = true |
| + case "replication": |
| + replicationSet = true |
| } |
| }) |
| @@ -513,7 +530,8 @@ func runRoute(args []string) { |
| " or: carlos route --host <host> (--grant | --revoke) <port:n|host-role> [--registry <db>] [--dry-run]\n" + |
| " or: carlos route --host <host> --hibernate[=false] [--registry <db>] [--dry-run]\n" + |
| " or: carlos route --host <host> --backing <unit|\"\"> [--registry <db>] [--dry-run]\n" + |
| - ` or: carlos route --host <host> --unit <name.service|-> [--registry <db>] [--dry-run]` |
| + " or: carlos route --host <host> --unit <name.service|-> [--registry <db>] [--dry-run]\n" + |
| + ` or: carlos route --host <host> --replication <stream|batched|daily|-> [--registry <db>] [--dry-run]` |
| if *host == "" && *app == "" { |
| log.Fatal(usage) |
| } |
| @@ -545,15 +563,23 @@ func runRoute(args []string) { |
| log.Fatal("route: --backing is mutually exclusive with --grant/--revoke and --hibernate — run them as two commands, so each meets its own rail alone. " + |
| "The order is not arbitrary either: --backing unit comes FIRST, because a grant onto a route that is not unit-backed is refused (it would materialize nothing)") |
| } |
| + // --replication is held to the same exclusions as the other single-route |
| + // axes, and for the reason recorded above --backing: the switch below |
| + // runs one arm, so a command naming two axes would write one of them and |
| + // drop the other on the floor without a word. |
| + if replicationSet && (grantAxis || hibernateSet || backingSet || unitSet) { |
| + log.Fatal("route: --replication is mutually exclusive with --grant/--revoke, --hibernate, --backing and --unit — run them as two commands, so each meets its own rail alone. " + |
| + "--hibernate in particular decides whether this route's class is read AT ALL (the activator replicates hibernating routes; /etc/litestream.yml covers the rest), so the two in one command would leave the answer depending on which arm ran") |
| + } |
| if unitSet && (grantAxis || hibernateSet || backingSet) { |
| log.Fatal("route: --unit is mutually exclusive with --grant/--revoke, --hibernate and --backing — run them as two commands, so each meets its own rail alone. " + |
| "--unit and --backing in particular are two different answers to the same question (who owns this route's process), and a command that wrote both would leave the row saying both") |
| } |
| - if (grantAxis || hibernateSet || backingSet || unitSet) && (*channel != "" || *environment != "" || *tunnel != "") { |
| - log.Fatal("route: --grant/--revoke/--hibernate/--backing/--unit cannot be combined with --channel, --environment or --tunnel — a capability grant and a repoint are different operations with different rails, and one command that did both would commit them together") |
| + if (grantAxis || hibernateSet || backingSet || unitSet || replicationSet) && (*channel != "" || *environment != "" || *tunnel != "") { |
| + log.Fatal("route: --grant/--revoke/--hibernate/--backing/--unit/--replication cannot be combined with --channel, --environment or --tunnel — a capability grant and a repoint are different operations with different rails, and one command that did both would commit them together") |
| } |
| - if (grantAxis || hibernateSet || backingSet || unitSet) && *host == "" { |
| - log.Fatal("route: --grant/--revoke/--hibernate/--backing/--unit require --host, never --app: grants, process backing and the owning unit are per route per box by design, and a bulk grant is precisely the operation nobody should be able to type by accident\n\n" + usage) |
| + if (grantAxis || hibernateSet || backingSet || unitSet || replicationSet) && *host == "" { |
| + log.Fatal("route: --grant/--revoke/--hibernate/--backing/--unit/--replication require --host, never --app: grants, process backing and the owning unit are per route per box by design, and a bulk grant is precisely the operation nobody should be able to type by accident\n\n" + usage) |
| } |
| // The token is checked BEFORE the registry is opened for writing: a typo |
| // should cost nothing and reach nothing. |
| @@ -598,11 +624,32 @@ func runRoute(args []string) { |
| log.Fatalf("route: unit %q: want a systemd service unit name (e.g. console.service), or \"-\" to clear the stamp", *unit) |
| } |
| } |
| + // --replication's value is checked at the flag, before the registry is |
| + // opened, on --unit's exact grounds: an unknown class reaches |
| + // replicationSyncInterval, which answers anything it does not recognise |
| + // with litestream's default — so `--replication dayly` would be written, |
| + // reported as done, and discovered as a route that quietly never left the |
| + // most expensive rung. "-" is the clear sentinel and resolves to "", the |
| + // column's own default; a bare `--replication ""` is refused, because the |
| + // doors that CLEAR say so out loud (see --unit). |
| + replicationClear := *replication == "-" |
| + resolvedReplication := *replication |
| + if replicationClear { |
| + resolvedReplication = "" |
| + } |
| + if replicationSet && !replicationClear { |
| + if *replication == "" { |
| + log.Fatalf(`route: --replication "" is refused — use "--replication -" to clear the rung, or name one: %s`, strings.Join(registry.ReplicationClasses(), ", ")) |
| + } |
| + if !registry.ValidReplication(*replication) { |
| + log.Fatalf("route: replication %q: want one of %s, or \"-\" to clear to the default", *replication, strings.Join(registry.ReplicationClasses(), ", ")) |
| + } |
| + } |
| if *tunnel != "" && *app != "" { |
| log.Fatal("route: --tunnel is only supported with --host — bulk --app tunnel repointing is not implemented\n\n" + usage) |
| } |
| - if *channel == "" && *environment == "" && *tunnel == "" && !grantAxis && !hibernateSet && !backingSet && !unitSet { |
| - log.Fatal(`route: at least one of --channel, --environment, --tunnel, --grant, --revoke, --hibernate, --backing or --unit is required. "" only means stable-by-default at route CREATION (carlos add) — ` + |
| + if *channel == "" && *environment == "" && *tunnel == "" && !grantAxis && !hibernateSet && !backingSet && !unitSet && !replicationSet { |
| + log.Fatal(`route: at least one of --channel, --environment, --tunnel, --grant, --revoke, --hibernate, --backing, --unit or --replication is required. "" only means stable-by-default at route CREATION (carlos add) — ` + |
| "this verb sets an explicit binding, and needs an explicit target to set it to.") |
| } |
| if *channel != "" && !release.ValidChannel(*channel) { |
| @@ -664,7 +711,7 @@ func runRoute(args []string) { |
| // The single-route axes short-circuit here, before the channel/environment |
| // machinery below: they write one column of one row, through one narrow |
| // setter, and share none of that batch's all-or-nothing plumbing. |
| - if grantAxis || hibernateSet || backingSet || unitSet { |
| + if grantAxis || hibernateSet || backingSet || unitSet || replicationSet { |
| rt, err := r.Get(*host) |
| if err != nil { |
| log.Fatal(err) |
| @@ -679,6 +726,8 @@ func runRoute(args []string) { |
| runRouteHibernate(r, *rt, *hibernate, *dryRun) |
| case backingSet: |
| runRouteBacking(r, *rt, *backing, *dryRun) |
| + case replicationSet: |
| + runRouteReplication(r, *rt, resolvedReplication, *dryRun) |
| default: |
| runRouteUnit(r, *rt, resolvedUnit, *dryRun) |
| } |
| @@ -1299,6 +1348,19 @@ func runRouteHibernate(r *registry.Registry, rt registry.Route, on, dryRun bool) |
| } |
| } |
| fmt.Printf("%s%s: hibernate %v -> %v\n", prefix, rt.Host, rt.Hibernate, on) |
| + // Said at the moment of typing, because the write does two things and |
| + // only one of them was asked for. registry.SetHibernate clears the |
| + // replication class when hibernation goes off — a rung is only read on |
| + // the activator's wake path, and a non-hibernating database is covered |
| + // by box-wide /etc/litestream.yml at its own interval, so keeping the |
| + // column would leave `carlos routes` printing a rung nothing honours. |
| + // An operator who turns hibernation back on has to name the rung again, |
| + // and being told that here is the difference between a decision and a |
| + // discovery. |
| + if !on && rt.Replication != "" { |
| + fmt.Printf(" replication %s cleared — the rung is read only on the activator's wake path, and this route no longer sleeps.\n", replicationWord(rt.Replication)) |
| + fmt.Printf(" its database is now covered by box-wide /etc/litestream.yml at that unit's own interval; re-`--hibernate` then set the rung again if you want it back\n") |
| + } |
| } |
| // runRouteBacking stamps who owns a route's process — the door design §5b |
| @@ -1477,6 +1539,112 @@ func unitWord(unit string) string { |
| return unit |
| } |
| +// runRouteReplication puts one route on a rung of the replication ladder. |
| +// |
| +// The ladder was built and honoured and had no door: three named rungs, one |
| +// of them reachable, and it was the most expensive one. Every route on the |
| +// fleet therefore ran on the every-second default, which for one tenant was |
| +// about 100x the S3 traffic the alternative would have cost — a 610 MiB |
| +// database, low write volume, compacted several times a day (discussion |
| +// #10). This verb is the smallest thing that closes that: the activator |
| +// already reads the column on wake and hands the interval to litestream, so |
| +// nothing new happens on the box. |
| +// |
| +// It is deliberately the OPERATOR's door and not a member-facing knob. The |
| +// loss window is a durability promise the platform makes — `daily` means up |
| +// to a day of writes lost with the box — and whether a member may choose it, |
| +// or whether it belongs to a plan tier, is a design question #10 filed |
| +// rather than answered. Nothing here forecloses either answer. |
| +// |
| +// Two rails, both structural, and they are one premise read whole. The class |
| +// is read in exactly one place — the activator's wake path — and |
| +// /etc/litestream.yml covers everything box-wide litestream is NOT told to |
| +// leave alone, at its own interval. What it is told to leave alone is |
| +// `r.Hibernate && r.DBPath != ""` (internal/ops/litestream.go), so BOTH |
| +// halves are required: a non-hibernating route, and a hibernating route |
| +// whose row names no database, are each swept into the box-wide config and |
| +// each would have the column written, reported as set, and read by nothing — |
| +// the same silent no-op the missing door was. |
| +// |
| +// The column also cannot outlive the first half. registry.SetHibernate |
| +// clears the rung when hibernation goes off, and Upsert refuses to carry one |
| +// onto a non-hibernating row, so `carlos routes` cannot come to print a rung |
| +// this rail would no longer let anyone set. |
| +// |
| +// The value's shape is checked at the flag, before the registry is opened. |
| +func runRouteReplication(r *registry.Registry, rt registry.Route, class string, dryRun bool) { |
| + if err := replicationRail(rt); err != nil { |
| + log.Fatalf("route: %s: %v", rt.Host, err) |
| + } |
| + prefix := "" |
| + if dryRun { |
| + prefix = colorFor(os.Stdout).warn("(dry run)") + " " |
| + } |
| + if rt.Replication == class { |
| + mutedf(os.Stdout, "%s: replication already %s — nothing to do\n", rt.Host, replicationWord(class)) |
| + return |
| + } |
| + if !dryRun { |
| + if err := r.SetReplication(rt.Host, class); err != nil { |
| + log.Fatalf("route: write: %v", err) |
| + } |
| + } |
| + fmt.Printf("%s%s: replication %s -> %s\n", prefix, rt.Host, replicationWord(rt.Replication), replicationWord(class)) |
| + // Said every time, because the write is not the change: the activator |
| + // hands the interval to litestream when it wakes the instance, so an |
| + // instance that is awake right now keeps its current interval until it |
| + // next sleeps and wakes. |
| + fmt.Printf(" takes effect on this instance's next wake — the sync interval is handed to litestream at wake, so an instance awake now keeps the old one until it sleeps\n") |
| +} |
| + |
| +// replicationRail is runRouteReplication's pair of refusals, split out as a |
| +// pure function for the reason unitRail and backingRail are: a rail nobody |
| +// can exercise without running the process is a rail nobody tests. |
| +func replicationRail(rt registry.Route) error { |
| + if !rt.Hibernate { |
| + return fmt.Errorf("route does not hibernate — the replication class is read only where the activator replicates, on wake, and /etc/litestream.yml (which covers every NON-hibernating database on this box) has its own interval. "+ |
| + "Setting a rung here would write a column nothing reads. Use `carlos route --host %s --hibernate` first if this route should be activator-managed", rt.Host) |
| + } |
| + // Hibernation alone was never the premise — the exclusion it stands on |
| + // is `r.Hibernate && r.DBPath != ""` (ops.FleetDBsWithSkipped), and |
| + // provision.routeFor's comment already records this trap from the other |
| + // side. A hibernating route whose row names no database is NOT excluded |
| + // from box-wide litestream: the dataDir glob claims its file anyway and |
| + // replicates it at that unit's interval, so a rung set here would be |
| + // written, reported by `carlos routes`, and honoured by nothing — the |
| + // same silent no-op the missing door was, reached through the door. |
| + // |
| + // Empty db_path on a hibernating row is not hypothetical: `carlos add` |
| + // defaults the path only for a single-upstream `instance` that is not |
| + // unit-backed, so a static or service route, or an instance record |
| + // carrying an upstream SET (provision.Record.DerivesPaths is false |
| + // there), hibernates with the column empty. |
| + if rt.DBPath == "" { |
| + return fmt.Errorf("route hibernates but its row names no database (db_path is empty) — box-wide litestream excludes a hibernating route only when it knows WHERE its database is (`r.Hibernate && r.DBPath != \"\"`, ops.FleetDBs), "+ |
| + "so this route's file is still being replicated at /etc/litestream.yml's own interval and a rung set here would be honoured by nothing. "+ |
| + "Give the row its path first (`carlos add --host %s --db <absolute path>`, which rewrites the existing route), then set the rung", rt.Host) |
| + } |
| + return nil |
| +} |
| + |
| +// replicationWord renders a replication column value for an operator — "" is |
| +// this column's default and a real answer, and printing it as "" reads as a |
| +// bug rather than as the rung every route was on before there was a door. |
| +func replicationWord(class string) string { |
| + if class == "" { |
| + return `"" (the default: change-driven ~1s sync, same as stream)` |
| + } |
| + switch class { |
| + case registry.ReplicationStream: |
| + return "stream (change-driven ~1s — loss window seconds)" |
| + case registry.ReplicationBatched: |
| + return "batched (at most every 5m — loss window minutes)" |
| + case registry.ReplicationDaily: |
| + return "daily (at wake and every 24h — loss window up to a day)" |
| + } |
| + return class |
| +} |
| + |
| // backingWord renders a backing column value for an operator — the empty |
| // string is a real value of this column, and printing it as "" reads as a |
| // bug rather than as an answer. |
diff --git a/docs/site/cli.md b/docs/site/cli.md| index 7c2997622..22a137a20 100644 |
| --- a/docs/site/cli.md |
| +++ b/docs/site/cli.md |
| @@ -452,9 +452,12 @@ point them at the steering hostname the app's Settings page names. |
| ### carlos routes |
| The app's routes on this deployment: where each one sends traffic, which |
| -channel it follows, and which config environment it is bound to. That last |
| -column is the one worth knowing about, because it decides which sealed |
| -secrets the route's process is delivered. |
| +channel it follows, which config environment it is bound to, and how often |
| +its database is replicated off the box. The environment column is the one |
| +worth knowing about, because it decides which sealed secrets the route's |
| +process is delivered. The replication column reads `default` until somebody |
| +sets a rung with `carlos route --replication`; the default syncs on every |
| +change. |
| ```sh |
| carlos routes --app hello |
| @@ -760,6 +763,36 @@ carlos route --host hello.example.com --channel edge |
| when idle. `--backing` and `--unit` are two answers to one question — who owns |
| this route's process — so the command will not let you write both. |
| +`--replication` picks how often the route's database is copied off the box: |
| +`stream` syncs on every change, about once a second; `batched` at most every |
| +five minutes; `daily` at wake and then once a day. The rung you pick is a |
| +trade, and it is the loss window you are trading. `daily` on a box that dies |
| +loses up to a day of writes. What it buys is real money on a large database |
| +that is written rarely — a 600 MB file compacted a few times a day costs the |
| +same in traffic whether or not anything changed, and that traffic was 97% of |
| +one day's replica writes across a whole fleet before anyone could turn it |
| +down. |
| + |
| +```sh |
| +carlos route --host hello.example.com --replication batched |
| +``` |
| + |
| +`--replication -` puts the route back on the default, which behaves exactly |
| +like `stream`. It only applies to routes that hibernate and whose row names a |
| +database: those are the ones the activator replicates, and everything else on |
| +the box is covered by the box's own replicator at its own interval. The |
| +command refuses both cases rather than writing a setting nothing would read — |
| +a route that does not hibernate, and one that hibernates with no `--db` path |
| +on it, which the box replicator cannot tell apart from any other file and so |
| +sweeps up anyway. A change takes effect the next time the instance wakes. |
| + |
| +The rung lives only as long as the sleeping does. Stopping a route from |
| +hibernating clears it — with `carlos route --hibernate=false`, and equally |
| +when an instance record turns hibernation off and the box converges — because |
| +from that moment the box replicator owns the database and the rung would name |
| +an interval nothing honours. The command says so when it clears one. Turning |
| +hibernation back on starts from the default again. |
| + |
| Repointing the channel is the half to be careful with. Adoption converges a |
| route onto whatever its channel currently points at, so moving the channel |
| changes what production serves on the next pass; the command resolves the |
diff --git a/internal/activator/activator.go b/internal/activator/activator.go| index 41f326995..f1b545544 100644 |
| --- a/internal/activator/activator.go |
| +++ b/internal/activator/activator.go |
| @@ -208,10 +208,15 @@ type Proc interface { |
| // Replication classes — the registry's routes.replication vocabulary |
| // ("" | stream | batched | daily), mapped to litestream sync intervals by |
| // replicationSyncInterval. |
| +// |
| +// Aliases, not copies: internal/registry owns the vocabulary now that |
| +// `carlos route --replication` writes the column against it (discussion |
| +// #10), and two independent spellings of the same three strings is exactly |
| +// how a setter and a reader drift into disagreeing about what `daily` means. |
| const ( |
| - replicationStream = "stream" // change-driven ~1s sync — loss window seconds |
| - replicationBatched = "batched" // sync at most every 5m — loss window minutes |
| - replicationDaily = "daily" // sync at wake + every 24h — loss window up to a day |
| + replicationStream = registry.ReplicationStream // change-driven ~1s sync — loss window seconds |
| + replicationBatched = registry.ReplicationBatched // sync at most every 5m — loss window minutes |
| + replicationDaily = registry.ReplicationDaily // sync at wake + every 24h — loss window up to a day |
| ) |
| // replicationSyncInterval maps a class to the litestream replica sync |
diff --git a/internal/console/apiroutes.go b/internal/console/apiroutes.go| index 4123bee93..bf16ca8dc 100644 |
| --- a/internal/console/apiroutes.go |
| +++ b/internal/console/apiroutes.go |
| @@ -61,6 +61,12 @@ type apiRoute struct { |
| Unit string `json:"unit,omitempty"` |
| Backing string `json:"backing,omitempty"` |
| Provisioned bool `json:"provisioned"` |
| + // Replication is the rung of the replication ladder this route's |
| + // database is replicated on — "" (the pre-ladder default), stream, |
| + // batched or daily. Additive, omitempty: an older client decodes the |
| + // absent field as "", which is exactly what every route carried before |
| + // there was a door onto the column (discussion #10). |
| + Replication string `json:"replication,omitempty"` |
| } |
| // apiRoutePath is one path route: a leading URL segment on Host that is |
| @@ -145,6 +151,7 @@ func (s *Server) handleAPIRoutes(w http.ResponseWriter, r *http.Request) { |
| Channel: rt.Channel, Environment: rt.Environment, Status: rt.Status, |
| Health: health[rt.Host], Hibernate: rt.Hibernate, Unit: rt.Unit, |
| Backing: rt.Backing, Provisioned: rt.Provisioned, |
| + Replication: rt.Replication, |
| }) |
| } |
| // Sorted by host: registry order is not stable across passes, and a |
diff --git a/internal/console/apiroutes_test.go b/internal/console/apiroutes_test.go| index e2a9a6a33..cdc036f45 100644 |
| --- a/internal/console/apiroutes_test.go |
| +++ b/internal/console/apiroutes_test.go |
| @@ -38,8 +38,13 @@ func routesFixture(t *testing.T, code string) (*consoleFixture, string) { |
| // exactly why `carlos instances list` could not answer for it. |
| {Host: "connect.go.tito.io", Kind: "instance", Status: "ready", App: "titogo", |
| Account: "bab", Addr: "127.0.0.1:9001", Channel: "canary/relay", Environment: "relay"}, |
| + // Hibernating, with a database path and a rung on it: the two halves |
| + // `carlos route --replication` requires, so this row is shaped the |
| + // way a real tuned tenant is. |
| {Host: "router-a.go.tito.io", Kind: "instance", Status: "ready", App: "titogo", |
| - Account: "bab", Addr: "127.0.0.1:9002", Channel: "stable"}, |
| + Account: "bab", Addr: "127.0.0.1:9002", Channel: "stable", |
| + Hibernate: true, DBPath: "/data/router-a.go.tito.io.db", |
| + Replication: registry.ReplicationDaily}, |
| {Host: "neighbour.example.com", Kind: "instance", Status: "ready", App: "kass", |
| Account: "bab", Addr: "127.0.0.1:9003"}, |
| {Host: "foreign.example.com", Kind: "instance", Status: "ready", App: "titogo", |
| @@ -137,6 +142,36 @@ func TestAPIRoutesReportsTheEnvironmentBinding(t *testing.T) { |
| } |
| } |
| +// TestAPIRoutesReportsTheReplicationRung: the rung is set on the box, by an |
| +// operator, with `carlos route --replication` — and the member reading |
| +// `carlos routes` is the only one who can tell whether the tenant they are |
| +// paying for is on the rung somebody intended. The field is `omitempty` on |
| +// the wire and nothing else in this repo reads it back out of JSON, so an |
| +// untested hop is one where the struct field could be deleted and every |
| +// route would print "default" with a green suite. |
| +func TestAPIRoutesReportsTheReplicationRung(t *testing.T) { |
| + fx, token := routesFixture(t, "code-routes-repl") |
| + status, out := getJSON(t, routesURL(fx, "titogo"), token) |
| + if status != http.StatusOK { |
| + t.Fatalf("GET routes = %d %v", status, out) |
| + } |
| + byHost := hostsOf(t, out) |
| + tuned, ok := byHost["router-a.go.tito.io"] |
| + if !ok { |
| + t.Fatalf("routes = %v, want the tuned route listed", out["routes"]) |
| + } |
| + if got := str(t, tuned, "replication"); got != registry.ReplicationDaily { |
| + t.Fatalf("replication = %q, want %q — the rung an operator set is unreadable to the member whose bill it is", got, registry.ReplicationDaily) |
| + } |
| + // A route on the pre-ladder default reports nothing rather than a rung it |
| + // is not on: the field is omitempty, and "" is what every route carried |
| + // before the column had a door. The CLI gives that absence its name |
| + // ("default", routes_cli.go's replicationCell) — the wire does not. |
| + if v, present := byHost["connect.go.tito.io"]["replication"]; present { |
| + t.Fatalf("an unset rung was sent as %v, want the field omitted", v) |
| + } |
| +} |
| + |
| // TestAPIRoutesListsOnlyThisAppsRoutes: the registry holds every tenant's |
| // rows, and the tenant boundary is the only thing standing between a |
| // caller and a neighbour's upstream addresses and unit names. |
diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go| index 7dcaa45a8..7f620aeae 100644 |
| --- a/internal/provision/reconcile.go |
| +++ b/internal/provision/reconcile.go |
| @@ -420,6 +420,11 @@ func shapeMatches(existing, want registry.Route) bool { |
| // columns there (the original message did app/kind/addr) leaves every |
| // refusal over any OTHER column looking identical and undiagnosable to the |
| // operator the refusal message exists for. |
| +// |
| +// `replication` is box-scoped and inexpressible in a record exactly like |
| +// Backing/Tunnel/Caps, and is DELIBERATELY not listed here — see the |
| +// asymmetry note above the removal loop below for why refusing adoption over |
| +// it would cost the fleet the backfill and buy nothing. |
| func shapeMismatch(existing, want registry.Route) string { |
| switch { |
| case existing.Backing != "": |
| @@ -808,13 +813,28 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg |
| convergeOwned(reg, existing, rec, cfg.DryRun) |
| // Absent means "this record does not speak to hibernation" — |
| // never a derivation stamped over an operator's own setting. |
| + // |
| + // Converging hibernation OFF also drops the route's replication |
| + // rung, inside registry.SetHibernate — a rung is read only on |
| + // the activator's wake path, so on a route the record has just |
| + // stopped hibernating it would describe nothing that happens |
| + // while `carlos routes` went on printing it. A record cannot |
| + // express the rung (it is box-scoped, like caps and tunnel), so |
| + // this pass cannot restore one either; the drop is LOGGED with |
| + // the flip rather than left for an operator to notice, because |
| + // it is the one operator-typed value a converge tick can |
| + // silently spend. |
| if rec.Hibernate != nil && existing.Hibernate != *rec.Hibernate { |
| + dropped := "" |
| + if !*rec.Hibernate && existing.Replication != "" { |
| + dropped = fmt.Sprintf(" (replication %q cleared: only a hibernating route's rung is read)", existing.Replication) |
| + } |
| if cfg.DryRun { |
| - log.Printf("provision: dry-run: would converge %s hibernate %v -> %v", rec.Host, existing.Hibernate, *rec.Hibernate) |
| + log.Printf("provision: dry-run: would converge %s hibernate %v -> %v%s", rec.Host, existing.Hibernate, *rec.Hibernate, dropped) |
| } else if err := reg.SetHibernate(rec.Host, *rec.Hibernate); err != nil { |
| log.Printf("provision: converge hibernate on %s: %v", rec.Host, err) |
| } else { |
| - log.Printf("provision: converged %s hibernate %v -> %v", rec.Host, existing.Hibernate, *rec.Hibernate) |
| + log.Printf("provision: converged %s hibernate %v -> %v%s", rec.Host, existing.Hibernate, *rec.Hibernate, dropped) |
| } |
| } |
| // seedIsKnown guards against stamping a transient manifest-read |
| @@ -919,6 +939,29 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg |
| // one unattended record-deletion away from registry.Delete's grant purge. |
| // That asymmetry predates this pass's adoption step; it is stated here so |
| // the adoption guard is not mistaken for a standing guarantee. |
| + // |
| + // `replication` (discussion #10) is the same KIND of column — box-scoped, |
| + // operator-typed, inexpressible in a record — and is deliberately exempt |
| + // from shapeMismatch rather than added to it. The three that are listed |
| + // there are listed for a reason that does not hold for this one: each |
| + // grants or relocates AUTHORITY over a route's process (a capability, a |
| + // process owner, an ingress path), and adoption is itself an authority |
| + // transfer, so handing bucket control a row that already carries one is |
| + // the thing worth stopping to look at. A replication rung grants nothing. |
| + // It names how often a database is copied off the box, and it survives |
| + // adoption unchanged: adoption's own write is SetProvisioned, the owned |
| + // branch converges through narrow setters that never touch this column, |
| + // and the one path that DOES clear it — registry.SetHibernate, when a |
| + // record turns hibernation off — is the honest answer rather than a loss, |
| + // since a non-hibernating route's rung is read by nobody. |
| + // |
| + // Listing it would mean an operator who tuned a large sleeping tenant's |
| + // rung had, by that act, opted the route out of the backfill this pass |
| + // exists to perform — permanently and invisibly, until someone thought to |
| + // clear a column to let a record adopt a row. That is the exact opposite |
| + // trade from the other three: it would strand precisely the hibernating |
| + // tenants the ladder was built for, in exchange for guarding an authority |
| + // this column does not carry. |
| if !readFailed { |
| for _, rt := range routes { |
| if !rt.Provisioned || !cfg.inScope(rt.Region, rt.Pool, everywhereRoute(rt)) { |
diff --git a/internal/provision/reconcile_test.go b/internal/provision/reconcile_test.go| index b23f39392..d71e9b35d 100644 |
| --- a/internal/provision/reconcile_test.go |
| +++ b/internal/provision/reconcile_test.go |
| @@ -602,6 +602,53 @@ func TestReconcileConvergesHibernateOnlyWhenExplicit(t *testing.T) { |
| } |
| } |
| +// Converging hibernation OFF drops the route's replication rung with it. |
| +// This is the quietest of the three doors that flip the column — no operator |
| +// types it, a record edit somewhere else does — and it is the one that made |
| +// `carlos route --replication`'s rail point-in-time: the rung is read only on |
| +// the activator's wake path, so the moment this pass stops a route sleeping, |
| +// box-wide /etc/litestream.yml owns that database and a stored `daily` is a |
| +// column `carlos routes` prints and nothing honours (discussion #10). |
| +// |
| +// A record cannot express a rung — it is box-scoped, like caps and tunnel — |
| +// so this pass can only drop one, never restore it. That is why the drop is |
| +// logged beside the flip rather than left to be discovered. |
| +func TestReconcileConvergingHibernateOffClearsTheReplicationRung(t *testing.T) { |
| + s, sqid := setup(t) |
| + rec := acmeRecord(sqid) |
| + on := true |
| + rec.Hibernate = &on |
| + if err := CreateRecord(ctx, s, rec); err != nil { |
| + t.Fatal(err) |
| + } |
| + reg := regWith(t) |
| + cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !mustGet(t, reg, "acme.go.tito.io").Hibernate { |
| + t.Fatal("setup: the minted route does not hibernate") |
| + } |
| + // The operator's own write, on the box, through the only door there is. |
| + if err := reg.SetReplication("acme.go.tito.io", registry.ReplicationDaily); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + off := false |
| + rec.Hibernate = &off |
| + putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + rt := mustGet(t, reg, "acme.go.tito.io") |
| + if rt.Hibernate { |
| + t.Fatal("setup: hibernate did not converge off") |
| + } |
| + if rt.Replication != "" { |
| + t.Errorf("replication = %q after the record stopped the route hibernating — the column now names an interval nothing reads", rt.Replication) |
| + } |
| +} |
| + |
| // StatusNamed is converged, so it carries Hibernate's tri-state for |
| // Hibernate's reason: explicit true reaches the row, explicit false CLEARS |
| // it, and absent is not an answer at all. |
diff --git a/internal/registry/registry.go b/internal/registry/registry.go| index 07baf89ce..05ede7c7f 100644 |
| --- a/internal/registry/registry.go |
| +++ b/internal/registry/registry.go |
| @@ -121,9 +121,21 @@ type Route struct { |
| // routing, and the edge has no reason to reload for one. |
| Caps string |
| - Hibernate bool |
| - Canary bool |
| - Replication string // "" | "stream" | "batched" | "daily" |
| + Hibernate bool |
| + Canary bool |
| + // Replication is the route's litestream sync class — the rung of the |
| + // replication ladder its database is replicated on. The vocabulary is |
| + // the four Replication* constants below, and internal/activator maps it |
| + // to a real sync interval on every wake. "" is not a fifth rung: it is |
| + // the pre-ladder default, and it deliberately buys what stream buys, so |
| + // a registry written before the column existed keeps its behaviour. |
| + // |
| + // Upsert PRESERVES a non-empty value (see its doc comment), which is |
| + // what stops the provisioning reconciler — which mints routes from |
| + // instance records, and records carry no replication class — from |
| + // silently returning an operator's `daily` route to the default within |
| + // seconds of the next serial bump. |
| + Replication string |
| NextDue int64 |
| StatusNamed bool // publish this route's app by name on the status page — the app's own opt-in |
| @@ -303,6 +315,45 @@ func ValidUnitName(name string) bool { |
| return name == "" || unitNameRe.MatchString(name) |
| } |
| +// The replication ladder — the vocabulary of the routes.replication column, |
| +// and the ONE place it is written down. internal/activator maps these to |
| +// litestream sync intervals; `carlos route --replication` is the door that |
| +// sets one. They lived only in the activator, unexported, for as long as |
| +// nothing could set the column (discussion #10): three named rungs, one of |
| +// them reachable, and it was the most expensive one. |
| +const ( |
| + // ReplicationDefault is the unset column. It buys what ReplicationStream |
| + // buys — change-driven ~1s sync — so a registry written before the ladder |
| + // existed behaves exactly as it did. It is the pre-ladder default rather |
| + // than a rung anyone chooses. |
| + ReplicationDefault = "" |
| + // ReplicationStream syncs on change, ~1s. Loss window: seconds. |
| + ReplicationStream = "stream" |
| + // ReplicationBatched syncs at most every 5m. Loss window: minutes. |
| + ReplicationBatched = "batched" |
| + // ReplicationDaily syncs at wake and every 24h. Loss window: up to a day. |
| + ReplicationDaily = "daily" |
| +) |
| + |
| +// ReplicationClasses lists the rungs an operator may set, in order of |
| +// decreasing durability. ReplicationDefault is absent on purpose: it is what |
| +// a route has before anyone chooses, and the way back to it is the CLI's |
| +// clear sentinel, not a name to type. |
| +func ReplicationClasses() []string { |
| + return []string{ReplicationStream, ReplicationBatched, ReplicationDaily} |
| +} |
| + |
| +// ValidReplication reports whether class is a value this column may hold. |
| +// "" is valid: it is the column's default and the state a cleared route |
| +// returns to, exactly as ValidUnitName accepts "". |
| +func ValidReplication(class string) bool { |
| + switch class { |
| + case ReplicationDefault, ReplicationStream, ReplicationBatched, ReplicationDaily: |
| + return true |
| + } |
| + return false |
| +} |
| + |
| func Open(path string) (*Registry, error) { |
| d, err := sql.Open("sqlite", path) |
| if err != nil { |
| @@ -704,12 +755,12 @@ func (r *Registry) Delete(host string) error { |
| return err |
| } |
| -// Upsert inserts or updates a route by host. Seven columns are deliberately |
| +// Upsert inserts or updates a route by host. Eight columns are deliberately |
| // not plain overwrites — `environment`, `caps`, `tunnel`, `provisioned`, |
| -// `seed`, `unit` and `fleet_box` all keep the existing row's value when the |
| -// incoming Route carries the zero one. |
| +// `seed`, `unit`, `fleet_box` and `replication` all keep the existing row's |
| +// value when the incoming Route carries the zero one. |
| // |
| -// The reason is the same for all seven, and it is structural: a full-row |
| +// The reason is the same for all eight, and it is structural: a full-row |
| // Upsert caller (`carlos add`, the domains reconciler's two mint paths, the |
| // provision pass's mint) builds its Route from what IT knows, and none of |
| // them knows about the columns another feature owns. `environment` was the |
| @@ -734,12 +785,46 @@ func (r *Registry) Delete(host string) error { |
| // the next promote would stamp X-Carlos-Version onto a console still serving |
| // the old build, which is issue #110 returning silently. |
| // |
| +// `replication` joined them when the ladder got a door (discussion #10) — |
| +// the same CASE as `environment`, but NOT, as an earlier draft of this |
| +// paragraph claimed, because the provisioning reconciler re-Upserts owned |
| +// rows. It does not: its only reg.Upsert is the mint branch (reconcile.go, |
| +// `case !ok`), and the owned branch converges through narrow setters |
| +// (convergeOwned, SetHibernate, SetSeed), exactly as the addrs/health |
| +// paragraph below already says. The domains reconciler's two Upserts are |
| +// mint-only too, guarded by byHost/hasApex. A reader who checks the old |
| +// claim finds it false and reasonably concludes the protection is |
| +// unnecessary, so it is worth stating what is actually true. |
| +// |
| +// The caller that really does Upsert over an EXISTING row is `carlos add` |
| +// (cli.go's runAdd — it prints "updated <host> (existing route)" when it |
| +// does). It builds its Route literal from flags alone, and it has no |
| +// replication flag: without this CASE, a re-add to restamp a unit or |
| +// repoint an addr would return an operator's `daily` route to the |
| +// every-second default, silently, at roughly 100x the S3 traffic the |
| +// setting was chosen to avoid. That is the same gap `environment` was |
| +// caught in and the same gap Addrs/Health/Steering are closed against at |
| +// the CLI's own door below. |
| +// |
| +// Against the two mint-only reconcilers the CASE is defensive rather than |
| +// load-bearing, and it stays that way deliberately: the rule for this |
| +// column is then identical to the seven beside it, so the NEXT full-row |
| +// Upsert caller — which will know nothing about replication either — is |
| +// covered before it is written rather than after it is diagnosed. |
| +// |
| // An explicit non-zero value still wins in every case; clearing keeps |
| // exactly one door per column, none of which goes through Upsert: |
| // SetEnvironment (`carlos route -environment -`), SetCaps (`carlos route |
| // -revoke`), SetTunnel (`carlos route -tunnel -`), SetUnit (`carlos route |
| -// -unit -`), SetSeed, SetFleetBox, and for `provisioned` the row's own |
| -// deletion. |
| +// -unit -`), SetReplication (`carlos route -replication -`), SetSeed, |
| +// SetFleetBox, and for `provisioned` the row's own deletion. |
| +// |
| +// `replication` is the one column with a SECOND clearing door, and it is not |
| +// an operator's: SetHibernate clears it when a route stops hibernating, |
| +// because a rung is read only on the activator's wake path and describes |
| +// nothing on a route that no longer sleeps. This statement's own |
| +// `excluded.hibernate = 0` arm is that same rule reaching the full-row |
| +// callers. See SetHibernate for the whole of it. |
| // |
| // `addrs` and `health` (multi-upstream routing, issue #152) are, by |
| // contrast, PLAIN overwrites — see their ON CONFLICT clause below — same as |
| @@ -784,6 +869,16 @@ func (r *Registry) Upsert(t Route) error { |
| if t.Kind == "" { |
| t.Kind = "instance" |
| } |
| + // The replication invariant, INSERT half (the ON CONFLICT half is the |
| + // `excluded.hibernate = 0` arm below): a non-empty class on a row that |
| + // does not hibernate is a column that lies. Nothing reads it there — |
| + // the activator's wake path is its only reader, and box-wide |
| + // /etc/litestream.yml covers every non-hibernating database at its own |
| + // interval — so a stored `daily` would be reported by `carlos routes` |
| + // while the database was streamed every second. See SetHibernate. |
| + if !t.Hibernate { |
| + t.Replication = "" |
| + } |
| b2i := func(b bool) int { |
| if b { |
| return 1 |
| @@ -800,12 +895,13 @@ ON CONFLICT(host) DO UPDATE SET |
| prev_version=excluded.prev_version, app=excluded.app, pool=excluded.pool, |
| region=excluded.region, channel=excluded.channel, |
| hibernate=excluded.hibernate, canary=excluded.canary, |
| - replication=excluded.replication, next_due=excluded.next_due, |
| + next_due=excluded.next_due, |
| backing=excluded.backing, status_named=excluded.status_named, |
| platform=excluded.platform, dns_status=excluded.dns_status, |
| dns_checked_at=excluded.dns_checked_at, |
| cert_expires_at=excluded.cert_expires_at, account=excluded.account, |
| environment=CASE WHEN excluded.environment = '' THEN routes.environment ELSE excluded.environment END, |
| + replication=CASE WHEN excluded.hibernate = 0 THEN '' WHEN excluded.replication = '' THEN routes.replication ELSE excluded.replication END, |
| caps=CASE WHEN excluded.caps = '' THEN routes.caps ELSE excluded.caps END, |
| tunnel=CASE WHEN excluded.tunnel = '' THEN routes.tunnel ELSE excluded.tunnel END, |
| provisioned=CASE WHEN excluded.provisioned = 0 THEN routes.provisioned ELSE excluded.provisioned END, |
| @@ -1045,6 +1141,39 @@ func (r *Registry) SetUnit(host, unit string) error { |
| return nil |
| } |
| +// SetReplication puts a route on a rung of the replication ladder, as a |
| +// narrow UPDATE for the reason SetChannel and SetUnit are: a |
| +// read-modify-write Upsert racing a deploy's SetVersions or the activator's |
| +// SetNextDue would clobber those columns. |
| +// |
| +// It validates, like SetUnit and unlike SetBacking: the value reaches |
| +// internal/activator's replicationSyncInterval, which answers an unknown |
| +// class with litestream's default — so `--replication dayly` would be |
| +// accepted, written, and discovered as a route that quietly never left the |
| +// most expensive rung. That is the failure the column's whole existence is |
| +// meant to end. |
| +// |
| +// "" is a legal value: it is the pre-ladder default a cleared route returns |
| +// to. It reaches here only from `carlos route --replication -`, never from |
| +// a full-row Upsert, which preserves the column instead. |
| +func (r *Registry) SetReplication(host, class string) error { |
| + if !ValidReplication(class) { |
| + return fmt.Errorf("replication %q: want one of %s, or \"\" for the default", class, strings.Join(ReplicationClasses(), ", ")) |
| + } |
| + res, err := r.sql.Exec(`UPDATE routes SET replication = ? WHERE host = ?`, class, host) |
| + if err != nil { |
| + return err |
| + } |
| + n, err := res.RowsAffected() |
| + if err != nil { |
| + return err |
| + } |
| + if n == 0 { |
| + return fmt.Errorf("no route for host %q", host) |
| + } |
| + return nil |
| +} |
| + |
| // SetUpstreams repoints a route's multi-upstream set as a narrow UPDATE — |
| // same rationale as SetChannel/SetUnit: a read-modify-write Upsert racing a |
| // deploy's SetVersions or the activator's SetNextDue would clobber those |
| @@ -1378,12 +1507,36 @@ func (r *Registry) SetVersions(host, ver, prev string) error { |
| } |
| // SetHibernate flips the per-host hibernation opt-in as a narrow UPDATE. |
| +// Turning hibernation OFF also clears `replication`, and that is the point |
| +// of doing it here rather than in each caller. The replication class is read |
| +// in exactly one place — the activator's wake path — and box-wide |
| +// /etc/litestream.yml deliberately covers every NON-hibernating database at |
| +// its own interval (internal/ops/litestream.go, ops.FleetDBsWithSkipped). |
| +// So the moment a route stops hibernating, a stored `daily` describes |
| +// nothing that happens: `carlos routes` would keep printing it while |
| +// litestream streamed the database every second, which is the lying column |
| +// the `carlos route --replication` rail (cli.go's replicationRail) exists to |
| +// prevent, merely relocated to a later moment. |
| +// |
| +// Both doors that turn hibernation off come through here — `carlos route |
| +// --hibernate=false` (cli.go's runRouteHibernate) and the provisioning |
| +// converge, which flips the column from the instance record on every pass — |
| +// and any door added later will too. The third writer, a full-row Upsert |
| +// carrying hibernate=0 (`carlos add` with no --hibernate), is covered by the |
| +// matching arm of that statement's ON CONFLICT clause. Between them the |
| +// invariant is total: a non-empty `replication` implies a hibernating row. |
| +// |
| +// Clearing rather than preserving-for-later is deliberate. A preserved class |
| +// is invisible state that changes behaviour weeks after it was typed, and |
| +// the operator turning hibernation back on is already at a prompt where |
| +// naming the rung again costs one command; runRouteHibernate prints what was |
| +// dropped so it is a fact they are told, not one they discover. |
| func (r *Registry) SetHibernate(host string, on bool) error { |
| - v := 0 |
| - if on { |
| - v = 1 |
| + if !on { |
| + _, err := r.sql.Exec(`UPDATE routes SET hibernate = 0, replication = '' WHERE host = ?`, host) |
| + return err |
| } |
| - _, err := r.sql.Exec(`UPDATE routes SET hibernate = ? WHERE host = ?`, v, host) |
| + _, err := r.sql.Exec(`UPDATE routes SET hibernate = 1 WHERE host = ?`, host) |
| return err |
| } |
diff --git a/internal/registry/replication_test.go b/internal/registry/replication_test.go| new file mode 100644 |
| index 000000000..ca459dbb1 |
| --- /dev/null |
| +++ b/internal/registry/replication_test.go |
| @@ -0,0 +1,219 @@ |
| +package registry |
| + |
| +import ( |
| + "strings" |
| + "testing" |
| +) |
| + |
| +// TestUpsertPreservesReplication is the fence that makes the new door worth |
| +// anything. The caller it stands against is `carlos add` re-run over a host |
| +// that already exists (cli.go's runAdd, which prints "updated ... (existing |
| +// route)"): it builds its Route literal from flags alone, and it has no |
| +// replication flag, so with a plain excluded.replication a re-add to restamp |
| +// a unit or repoint an addr would return an operator's `daily` route to the |
| +// every-second default — silently, and at roughly 100x the S3 traffic the |
| +// setting was chosen to avoid (discussion #10). Same shape, same reason, as |
| +// `environment`. |
| +// |
| +// NOT the provisioning reconciler, whatever the first draft of this comment |
| +// said: its only reg.Upsert is the mint branch (reconcile.go, `case !ok`, |
| +// where there is no existing row to clobber), and the owned branch converges |
| +// through narrow setters. The domains reconciler's two Upserts are mint-only |
| +// as well. The re-Upsert below is therefore a stand-in for `carlos add`'s |
| +// shape — a full-row write that knows nothing about this column — which is |
| +// the shape any future full-row caller will have too. |
| +func TestUpsertPreservesReplication(t *testing.T) { |
| + r := openTestRegistry(t) |
| + host := "seapointish.example.com" |
| + if err := r.Upsert(Route{Host: host, Kind: "instance", App: "seapointish", Hibernate: true}); err != nil { |
| + t.Fatalf("upsert: %v", err) |
| + } |
| + if err := r.SetReplication(host, ReplicationDaily); err != nil { |
| + t.Fatalf("SetReplication: %v", err) |
| + } |
| + |
| + // A full-row Upsert built by a caller that knows nothing about this |
| + // column — `carlos add`'s shape, and the reconciler's mint's shape. |
| + if err := r.Upsert(Route{Host: host, Kind: "instance", App: "seapointish", Hibernate: true, Channel: "edge"}); err != nil { |
| + t.Fatalf("re-upsert: %v", err) |
| + } |
| + rt, err := r.Get(host) |
| + if err != nil || rt == nil { |
| + t.Fatalf("get: %+v %v", rt, err) |
| + } |
| + if rt.Replication != ReplicationDaily { |
| + t.Fatalf("replication = %q after a full-row re-Upsert, want %q — a re-add silently returned the route to the most expensive rung", rt.Replication, ReplicationDaily) |
| + } |
| + if rt.Channel != "edge" { |
| + t.Fatalf("channel = %q, want the re-Upsert's own value: preserving replication must not freeze the columns a full-row caller DOES own", rt.Channel) |
| + } |
| +} |
| + |
| +// TestUpsertHonoursAnExplicitReplication: preservation is for the zero value |
| +// only. A caller that carries a class still wins, exactly as with environment. |
| +// Both rows hibernate, because a rung on a row that does not is a state this |
| +// table refuses to hold — see TestReplicationImpliesHibernation. |
| +func TestUpsertHonoursAnExplicitReplication(t *testing.T) { |
| + r := openTestRegistry(t) |
| + host := "a.example.com" |
| + if err := r.Upsert(Route{Host: host, Hibernate: true, Replication: ReplicationDaily}); err != nil { |
| + t.Fatalf("upsert: %v", err) |
| + } |
| + if err := r.Upsert(Route{Host: host, Hibernate: true, Replication: ReplicationBatched}); err != nil { |
| + t.Fatalf("re-upsert: %v", err) |
| + } |
| + rt, _ := r.Get(host) |
| + if rt.Replication != ReplicationBatched { |
| + t.Fatalf("replication = %q, want %q", rt.Replication, ReplicationBatched) |
| + } |
| +} |
| + |
| +// TestReplicationImpliesHibernation is the invariant that stops the column |
| +// from lying after the fact. `carlos route --replication` refuses a route |
| +// that does not hibernate, because the rung is read on the activator's wake |
| +// path and nowhere else — but that rail is point-in-time, and hibernation is |
| +// flipped afterwards by three writers: `carlos route --hibernate=false`, the |
| +// provisioning converge reading an instance record, and a full-row Upsert |
| +// from `carlos add`. Without this invariant a `daily` route becomes a |
| +// non-hibernating route that `carlos routes` still reports as `daily`, while |
| +// box-wide /etc/litestream.yml replicates its database at its own interval: |
| +// the lying column the rail exists to prevent, relocated to a later moment. |
| +// |
| +// Both halves are pinned here because they are two different statements — |
| +// one in SetHibernate's UPDATE, one in Upsert's INSERT and ON CONFLICT — and |
| +// a fix to either alone leaves a door open. |
| +func TestReplicationImpliesHibernation(t *testing.T) { |
| + r := openTestRegistry(t) |
| + |
| + // The narrow setter: `carlos route --hibernate=false` and the converge |
| + // pass both arrive here. |
| + host := "sleeper.example.com" |
| + if err := r.Upsert(Route{Host: host, Kind: "instance", App: "sleeper", Hibernate: true, |
| + DBPath: "/data/sleeper.example.com.db"}); err != nil { |
| + t.Fatalf("upsert: %v", err) |
| + } |
| + if err := r.SetReplication(host, ReplicationDaily); err != nil { |
| + t.Fatalf("SetReplication: %v", err) |
| + } |
| + if err := r.SetHibernate(host, false); err != nil { |
| + t.Fatalf("SetHibernate off: %v", err) |
| + } |
| + rt, err := r.Get(host) |
| + if err != nil || rt == nil { |
| + t.Fatalf("get: %+v %v", rt, err) |
| + } |
| + if rt.Replication != "" { |
| + t.Fatalf("replication = %q on a route that stopped hibernating — box-wide litestream owns this database now, and the column claims otherwise", rt.Replication) |
| + } |
| + // Turning hibernation back ON does not resurrect the rung: a preserved |
| + // class is invisible state that changes behaviour weeks later, and the |
| + // operator is at a prompt where naming it again costs one command. |
| + if err := r.SetHibernate(host, true); err != nil { |
| + t.Fatalf("SetHibernate on: %v", err) |
| + } |
| + if rt, _ = r.Get(host); rt.Replication != "" { |
| + t.Fatalf("re-hibernating restored %q — the rung is set, not remembered", rt.Replication) |
| + } |
| + |
| + // The full-row write: `carlos add` with no --hibernate over an existing |
| + // tuned row. It turns hibernation off (hibernate is a plain overwrite), |
| + // so the preserved-when-empty CASE must not hand the new row a rung. |
| + if err := r.SetReplication(host, ReplicationBatched); err != nil { |
| + t.Fatalf("SetReplication batched: %v", err) |
| + } |
| + if err := r.Upsert(Route{Host: host, Kind: "instance", App: "sleeper", |
| + DBPath: "/data/sleeper.example.com.db"}); err != nil { |
| + t.Fatalf("re-upsert awake: %v", err) |
| + } |
| + if rt, _ = r.Get(host); rt.Replication != "" { |
| + t.Fatalf("replication = %q after an Upsert that turned hibernation off", rt.Replication) |
| + } |
| + |
| + // And on INSERT, where there is no ON CONFLICT clause to do it. |
| + fresh := "awake.example.com" |
| + if err := r.Upsert(Route{Host: fresh, Kind: "instance", App: "awake", Replication: ReplicationDaily}); err != nil { |
| + t.Fatalf("insert: %v", err) |
| + } |
| + if rt, _ = r.Get(fresh); rt.Replication != "" { |
| + t.Fatalf("a fresh non-hibernating row was inserted carrying %q", rt.Replication) |
| + } |
| +} |
| + |
| +// TestSetReplicationValidates: the value reaches internal/activator's |
| +// replicationSyncInterval, which answers anything it does not recognise with |
| +// litestream's default — so an unvalidated write is a route that quietly |
| +// never leaves the most expensive rung. SetUnit validates for the same |
| +// reason; SetBacking does not need to, because its two legal values are |
| +// compared at every door. |
| +func TestSetReplicationValidates(t *testing.T) { |
| + r := openTestRegistry(t) |
| + host := "a.example.com" |
| + if err := r.Upsert(Route{Host: host}); err != nil { |
| + t.Fatalf("upsert: %v", err) |
| + } |
| + err := r.SetReplication(host, "dayly") |
| + if err == nil { |
| + t.Fatal("SetReplication accepted an unknown class") |
| + } |
| + for _, want := range ReplicationClasses() { |
| + if !strings.Contains(err.Error(), want) { |
| + t.Errorf("refusal %q does not list %q", err, want) |
| + } |
| + } |
| + rt, _ := r.Get(host) |
| + if rt.Replication != "" { |
| + t.Fatalf("a refused write still landed: %q", rt.Replication) |
| + } |
| + |
| + // "" is legal: it is the column's default and the state a cleared route |
| + // returns to. |
| + if err := r.SetReplication(host, ReplicationBatched); err != nil { |
| + t.Fatalf("SetReplication batched: %v", err) |
| + } |
| + if err := r.SetReplication(host, ReplicationDefault); err != nil { |
| + t.Fatalf("SetReplication to the default: %v", err) |
| + } |
| + rt, _ = r.Get(host) |
| + if rt.Replication != "" { |
| + t.Fatalf("clear left %q", rt.Replication) |
| + } |
| +} |
| + |
| +// TestSetReplicationUnknownHost: like SetChannel and SetUnit, this is a |
| +// setter a human types a hostname into, so an unknown host is an error rather |
| +// than a silent no-op. |
| +func TestSetReplicationUnknownHost(t *testing.T) { |
| + r := openTestRegistry(t) |
| + err := r.SetReplication("nope.example.com", ReplicationDaily) |
| + if err == nil || !strings.Contains(err.Error(), "nope.example.com") { |
| + t.Fatalf("SetReplication(unknown host) = %v, want an error naming the host", err) |
| + } |
| +} |
| + |
| +// TestValidReplicationVocabulary pins the set itself. It lives here now |
| +// rather than unexported in internal/activator, because a setter and a reader |
| +// with two independent spellings of the same three strings is how they drift |
| +// into disagreeing about what `daily` means. |
| +func TestValidReplicationVocabulary(t *testing.T) { |
| + for _, ok := range []string{"", "stream", "batched", "daily"} { |
| + if !ValidReplication(ok) { |
| + t.Errorf("ValidReplication(%q) = false", ok) |
| + } |
| + } |
| + for _, bad := range []string{"dayly", "STREAM", "hourly", "none", "5m"} { |
| + if ValidReplication(bad) { |
| + t.Errorf("ValidReplication(%q) = true", bad) |
| + } |
| + } |
| + // The default is deliberately absent from the offered list: it is what a |
| + // route has before anyone chooses, and the way back to it is the CLI's |
| + // clear sentinel, not a name to type. |
| + for _, c := range ReplicationClasses() { |
| + if c == ReplicationDefault { |
| + t.Errorf("ReplicationClasses() offers the default as a rung to type") |
| + } |
| + } |
| + if len(ReplicationClasses()) != 3 { |
| + t.Errorf("ReplicationClasses() = %v, want the three rungs", ReplicationClasses()) |
| + } |
| +} |
diff --git a/route_replication_test.go b/route_replication_test.go| new file mode 100644 |
| index 000000000..ec6fc74e0 |
| --- /dev/null |
| +++ b/route_replication_test.go |
| @@ -0,0 +1,333 @@ |
| +package main |
| + |
| +// `carlos route --replication` — the door the replication ladder did not |
| +// have. The ladder was built and honoured and unreachable: three named |
| +// rungs, one of them reachable, and it was the most expensive one |
| +// (discussion #10). |
| + |
| +import ( |
| + "os/exec" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| +) |
| + |
| +// TestReplicationRailRefusesNonHibernatingRoutes pins the first of this |
| +// verb's two refusals, as a pure function for the reason unitRail and |
| +// backingRail are: the class is read in exactly one place — the activator's |
| +// wake path — and /etc/litestream.yml covers everything the box-wide |
| +// replicator is not told to leave alone, at its own interval. On a route that |
| +// does not hibernate the column would be written, reported as set, and read |
| +// by nothing, which is the same silent no-op the missing door was. |
| +func TestReplicationRailRefusesNonHibernatingRoutes(t *testing.T) { |
| + awake := registry.Route{ |
| + Host: "console.carloku.com", Kind: "instance", App: "console", |
| + Addr: "127.0.0.1:8090", DBPath: "/data/console.carloku.com.db", |
| + } |
| + err := replicationRail(awake) |
| + if err == nil { |
| + t.Fatal("replicationRail(non-hibernating route) accepted; want a refusal") |
| + } |
| + for _, want := range []string{"hibernate", "litestream"} { |
| + if !strings.Contains(err.Error(), want) { |
| + t.Errorf("refusal %q does not mention %q — the operator has to be told which replicator owns this database instead", err, want) |
| + } |
| + } |
| + |
| + hibernating := awake |
| + hibernating.Hibernate = true |
| + if err := replicationRail(hibernating); err != nil { |
| + t.Fatalf("replicationRail(hibernating route) = %v, want accepted", err) |
| + } |
| +} |
| + |
| +// TestReplicationRailRefusesAHibernatingRouteWithNoDatabase pins the second |
| +// half of the same premise, which the first version of this rail missed. |
| +// Box-wide litestream excludes a route on `r.Hibernate && r.DBPath != ""` |
| +// (ops.FleetDBsWithSkipped) — hibernation ALONE excludes nothing. A |
| +// hibernating route whose row names no database is claimed by the dataDir |
| +// glob like any other file and replicated at /etc/litestream.yml's interval, |
| +// so a rung set on it would be written, reported by `carlos routes`, and |
| +// honoured by nobody: the exact failure the rail exists to refuse, reached |
| +// through the rail. |
| +// |
| +// Not hypothetical: `carlos add` defaults db_path only for a single-upstream |
| +// non-unit `instance`, so a static or service route, or an instance record |
| +// carrying an upstream set (provision.Record.DerivesPaths is false there), |
| +// hibernates with the column empty. |
| +func TestReplicationRailRefusesAHibernatingRouteWithNoDatabase(t *testing.T) { |
| + pathless := registry.Route{ |
| + Host: "static.example.com", Kind: "static", App: "site", |
| + Hibernate: true, |
| + } |
| + err := replicationRail(pathless) |
| + if err == nil { |
| + t.Fatal("replicationRail(hibernating route with no db_path) accepted; want a refusal") |
| + } |
| + for _, want := range []string{"db_path", "--db"} { |
| + if !strings.Contains(err.Error(), want) { |
| + t.Errorf("refusal %q does not mention %q — the operator has to be told what is missing and which door supplies it", err, want) |
| + } |
| + } |
| +} |
| + |
| +// TestReplicationWordNamesTheDefault: "" is this column's value on every |
| +// route that predates the door, it behaves exactly like stream, and printing |
| +// it as an empty string reads as a bug rather than as the answer it is. |
| +func TestReplicationWordNamesTheDefault(t *testing.T) { |
| + if got := replicationWord(""); !strings.Contains(got, "default") { |
| + t.Errorf(`replicationWord("") = %q, want it to name the default`, got) |
| + } |
| + for _, class := range registry.ReplicationClasses() { |
| + got := replicationWord(class) |
| + if !strings.Contains(got, class) { |
| + t.Errorf("replicationWord(%q) = %q, want it to name the rung", class, got) |
| + } |
| + if !strings.Contains(got, "loss window") { |
| + t.Errorf("replicationWord(%q) = %q, want the loss window spelled out — that is the whole trade", class, got) |
| + } |
| + } |
| +} |
| + |
| +// TestReplicationCellNamesTheDefault: the console-side listing has the same |
| +// duty. Unlike ENVIRONMENT, whose empty value is dashed, this one is named: |
| +// it is the column where "no answer" cost one tenant 97% of a day's replica |
| +// traffic. |
| +func TestReplicationCellNamesTheDefault(t *testing.T) { |
| + if got := replicationCell(""); got != "default" { |
| + t.Errorf(`replicationCell("") = %q, want "default"`, got) |
| + } |
| + if got := replicationCell(registry.ReplicationDaily); got != "daily" { |
| + t.Errorf("replicationCell(daily) = %q", got) |
| + } |
| +} |
| + |
| +// TestRouteReplicationSetsAndClearsTheRung drives the whole door as a |
| +// subprocess, the shape it ships in: the routes that need a cheaper rung |
| +// already exist on live boxes, so this stamps a LIVE registry row rather than |
| +// needing the route recreated. |
| +// |
| +// It pins the failure a careless door would cause as much as the one the |
| +// missing door did: replicationSyncInterval answers an unknown class with |
| +// litestream's default, so an accepted typo is a route that quietly never |
| +// left the most expensive rung. |
| +func TestRouteReplicationSetsAndClearsTheRung(t *testing.T) { |
| + bin := buildCarlos(t) |
| + tmp := t.TempDir() |
| + scratch := t.TempDir() |
| + regPath := filepath.Join(tmp, "registry.db") |
| + |
| + run := func(args ...string) (string, error) { |
| + t.Helper() |
| + cmd := exec.Command(bin, args...) |
| + cmd.Env = carlosTestEnv(t) |
| + // pick() walks cwd ancestors for .carlos/config, so the subprocess |
| + // must not run from the real checkout. |
| + cmd.Dir = scratch |
| + out, err := cmd.CombinedOutput() |
| + return string(out), err |
| + } |
| + replicationOf := func(host string) string { |
| + t.Helper() |
| + reg, err := registry.Open(regPath) |
| + if err != nil { |
| + t.Fatalf("open registry: %v", err) |
| + } |
| + defer reg.Close() |
| + rt, err := reg.Get(host) |
| + if err != nil || rt == nil { |
| + t.Fatalf("get %s: %+v, %v", host, rt, err) |
| + } |
| + return rt.Replication |
| + } |
| + |
| + // A hibernating instance as one exists on a box today: a large database, |
| + // low write volume, on the every-second default because nothing could say |
| + // otherwise. |
| + if out, err := run("add", "-registry", regPath, "-app", "seapointish", |
| + "-socket", filepath.Join(tmp, "s.sock"), "-hibernate", "seapointish.example.com"); err != nil { |
| + t.Fatalf("add route: %v\n%s", err, out) |
| + } |
| + if got := replicationOf("seapointish.example.com"); got != "" { |
| + t.Fatalf("a fresh route already carries a rung: %q", got) |
| + } |
| + |
| + // A typo must cost nothing and reach nothing. |
| + out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", "-replication", "dayly") |
| + if err == nil { |
| + t.Fatalf("route --replication dayly: want non-zero exit, got 0\n%s", out) |
| + } |
| + if got := replicationOf("seapointish.example.com"); got != "" { |
| + t.Fatalf("a refused --replication still wrote %q", got) |
| + } |
| + |
| + // A bare "" is refused rather than silently treated as a clear: the doors |
| + // that clear say so out loud. |
| + if out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", "-replication", ""); err == nil { |
| + t.Fatalf("route --replication \"\": want non-zero exit, got 0\n%s", out) |
| + } |
| + |
| + // -dry-run runs the rail and prints the plan, writing nothing. |
| + out, err = run("route", "-registry", regPath, "-host", "seapointish.example.com", |
| + "-replication", "daily", "-dry-run") |
| + if err != nil { |
| + t.Fatalf("route --replication --dry-run: %v\n%s", err, out) |
| + } |
| + if !strings.Contains(out, "daily") { |
| + t.Errorf("dry run output = %q, want it to name the rung", out) |
| + } |
| + if got := replicationOf("seapointish.example.com"); got != "" { |
| + t.Fatalf("--dry-run wrote %q", got) |
| + } |
| + |
| + // The real write. |
| + if out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", |
| + "-replication", "daily"); err != nil { |
| + t.Fatalf("route --replication daily: %v\n%s", err, out) |
| + } |
| + if got := replicationOf("seapointish.example.com"); got != registry.ReplicationDaily { |
| + t.Fatalf("replication = %q, want %q", got, registry.ReplicationDaily) |
| + } |
| + |
| + // And the way back, which is the sentinel rather than a name to type. |
| + if out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", |
| + "-replication", "-"); err != nil { |
| + t.Fatalf("route --replication -: %v\n%s", err, out) |
| + } |
| + if got := replicationOf("seapointish.example.com"); got != "" { |
| + t.Fatalf("--replication - left %q, want the default", got) |
| + } |
| + |
| + // The listing has to answer the question the setter just decided, or an |
| + // operator cannot see what they set — which was half of #10's complaint. |
| + if out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", |
| + "-replication", "batched"); err != nil { |
| + t.Fatalf("route --replication batched: %v\n%s", err, out) |
| + } |
| + out, err = run("instances", "-registry", regPath) |
| + if err != nil { |
| + t.Fatalf("instances: %v\n%s", err, out) |
| + } |
| + if !strings.Contains(out, "REPL") || !strings.Contains(out, "batched") { |
| + t.Errorf("`carlos instances` does not show the rung:\n%s", out) |
| + } |
| +} |
| + |
| +// TestRouteReplicationRefusesCombinationWithOtherAxes: the dispatch runs one |
| +// arm, so a command naming two axes would write one and drop the other on the |
| +// floor without a word — the silent partial --backing's exclusions were added |
| +// for. --app is refused outright: a bulk durability change is precisely the |
| +// operation nobody should be able to type by accident. |
| +func TestRouteReplicationRefusesCombinationWithOtherAxes(t *testing.T) { |
| + bin := buildCarlos(t) |
| + tmp := t.TempDir() |
| + scratch := t.TempDir() |
| + regPath := filepath.Join(tmp, "registry.db") |
| + |
| + run := func(args ...string) (string, error) { |
| + t.Helper() |
| + cmd := exec.Command(bin, args...) |
| + cmd.Env = carlosTestEnv(t) |
| + cmd.Dir = scratch |
| + out, err := cmd.CombinedOutput() |
| + return string(out), err |
| + } |
| + if out, err := run("add", "-registry", regPath, "-app", "a", |
| + "-socket", filepath.Join(tmp, "a.sock"), "-hibernate", "a.example.com"); err != nil { |
| + t.Fatalf("add: %v\n%s", err, out) |
| + } |
| + for _, extra := range [][]string{ |
| + {"-hibernate=false"}, |
| + {"-channel", "edge"}, |
| + {"-unit", "a.service"}, |
| + } { |
| + args := append([]string{"route", "-registry", regPath, "-host", "a.example.com", "-replication", "daily"}, extra...) |
| + if out, err := run(args...); err == nil { |
| + t.Errorf("route %v: want a refusal, got 0\n%s", extra, out) |
| + } |
| + } |
| + if out, err := run("route", "-registry", regPath, "-app", "a", "-replication", "daily"); err == nil { |
| + t.Errorf("route --app --replication: want a refusal, got 0\n%s", out) |
| + } |
| +} |
| + |
| +// TestRouteHibernateOffClearsTheRung: the rail is point-in-time, and until |
| +// this landed nothing held it afterwards. A route set to `daily` could stop |
| +// hibernating — by this verb, by `carlos add` with no --hibernate, or by a |
| +// converge pass reading an instance record — and `carlos routes` went on |
| +// printing `daily` while box-wide /etc/litestream.yml replicated the database |
| +// at its own interval. A column that lies is the failure the door was built |
| +// to end, so it must not survive the flip that makes it false. |
| +// |
| +// The clearing itself lives in registry.SetHibernate, where every door that |
| +// turns hibernation off passes; what is pinned here is the operator-facing |
| +// half — the flip works through the real binary, and it SAYS what it spent, |
| +// because the rung is the one operator-typed value this command drops |
| +// without being asked to. |
| +func TestRouteHibernateOffClearsTheRung(t *testing.T) { |
| + bin := buildCarlos(t) |
| + tmp := t.TempDir() |
| + scratch := t.TempDir() |
| + regPath := filepath.Join(tmp, "registry.db") |
| + |
| + run := func(args ...string) (string, error) { |
| + t.Helper() |
| + cmd := exec.Command(bin, args...) |
| + cmd.Env = carlosTestEnv(t) |
| + cmd.Dir = scratch |
| + out, err := cmd.CombinedOutput() |
| + return string(out), err |
| + } |
| + rowOf := func(host string) (string, bool) { |
| + t.Helper() |
| + reg, err := registry.Open(regPath) |
| + if err != nil { |
| + t.Fatalf("open registry: %v", err) |
| + } |
| + defer reg.Close() |
| + rt, err := reg.Get(host) |
| + if err != nil || rt == nil { |
| + t.Fatalf("get %s: %+v, %v", host, rt, err) |
| + } |
| + return rt.Replication, rt.Hibernate |
| + } |
| + |
| + host := "seapointish.example.com" |
| + if out, err := run("add", "-registry", regPath, "-app", "seapointish", |
| + "-socket", filepath.Join(tmp, "s.sock"), "-hibernate", host); err != nil { |
| + t.Fatalf("add route: %v\n%s", err, out) |
| + } |
| + if out, err := run("route", "-registry", regPath, "-host", host, "-replication", "daily"); err != nil { |
| + t.Fatalf("route --replication daily: %v\n%s", err, out) |
| + } |
| + if class, _ := rowOf(host); class != registry.ReplicationDaily { |
| + t.Fatalf("setup: replication = %q, want %q", class, registry.ReplicationDaily) |
| + } |
| + |
| + out, err := run("route", "-registry", regPath, "-host", host, "-hibernate=false") |
| + if err != nil { |
| + t.Fatalf("route --hibernate=false: %v\n%s", err, out) |
| + } |
| + class, hib := rowOf(host) |
| + if hib { |
| + t.Fatalf("setup: the route still hibernates\n%s", out) |
| + } |
| + if class != "" { |
| + t.Fatalf("replication = %q on a route that no longer hibernates — `carlos routes` would print a rung the box replicator does not honour", class) |
| + } |
| + // Told, not discovered. |
| + for _, want := range []string{"replication", "cleared"} { |
| + if !strings.Contains(out, want) { |
| + t.Errorf("--hibernate=false output does not mention %q:\n%s", want, out) |
| + } |
| + } |
| + |
| + // And the door is shut behind it: the rung cannot be re-set while the |
| + // route is awake, which is what makes the clear an honest state rather |
| + // than a gap between two commands. |
| + if out, err := run("route", "-registry", regPath, "-host", host, "-replication", "daily"); err == nil { |
| + t.Errorf("route --replication on a woken route: want a refusal, got 0\n%s", out) |
| + } |
| +} |
diff --git a/routes_cli.go b/routes_cli.go| index 209d3adb9..6e5a3a9b7 100644 |
| --- a/routes_cli.go |
| +++ b/routes_cli.go |
| @@ -58,28 +58,38 @@ func runRoutes(args []string) { |
| } |
| shown := 0 |
| w := newTable(stdout) |
| - fmt.Fprintln(w, "HOST\tKIND\tUPSTREAM\tCHANNEL\tENVIRONMENT\tSTATUS\tHEALTH") |
| + fmt.Fprintln(w, "HOST\tKIND\tUPSTREAM\tCHANNEL\tENVIRONMENT\tREPLICATION\tSTATUS\tHEALTH") |
| for _, r := range rows { |
| if *match != "" && !strings.Contains(r.Host, *match) { |
| continue |
| } |
| shown++ |
| - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", |
| + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", |
| r.Host, dashIfEmpty(r.Kind), dashIfEmpty(r.Upstream), dashIfEmpty(r.Channel), |
| // An empty ENVIRONMENT is a real answer — the route takes the |
| // app's default bundle — so it prints as a dash rather than a |
| // blank cell a reader would take for missing data. |
| - dashIfEmpty(r.Environment), dashIfEmpty(r.Status), dashIfEmpty(r.Health)) |
| + dashIfEmpty(r.Environment), |
| + // REPLICATION reads the same way: empty is the pre-ladder |
| + // default (change-driven ~1s sync), which is what every route |
| + // carried until `carlos route --replication` gave the column a |
| + // door (discussion #10). It prints "default" rather than a |
| + // dash, because unlike ENVIRONMENT the empty value has a name |
| + // and a behaviour worth saying out loud — this is the column |
| + // where "no answer" cost one tenant 97% of a day's replica |
| + // traffic. |
| + replicationCell(r.Replication), |
| + dashIfEmpty(r.Status), dashIfEmpty(r.Health)) |
| // Path routes indented under the hostname whose traffic they split. |
| // A flipped prefix set has to be visible from the command operators |
| // already use, or a migration is invisible until something breaks. |
| // |
| // KIND says "path" and UPSTREAM names the TARGET ROUTE rather than |
| // an address, because that is all a path row is: a pointer, whose |
| - // channel, environment and health are the target's own row's, listed |
| - // wherever that route appears. Dashes rather than blanks in the |
| - // remaining columns, for the reason ENVIRONMENT prints one above — |
| - // a blank cell reads as data this client failed to fetch. |
| + // channel, environment, replication and health are the target's own |
| + // row's, listed wherever that route appears. Dashes rather than |
| + // blanks in the remaining columns, for the reason ENVIRONMENT prints |
| + // one above — a blank cell reads as data this client failed to fetch. |
| // |
| // Read-only, here and everywhere in the CLI: the tenant's own |
| // platform publishes these wholesale and is their source of truth, |
| @@ -88,7 +98,7 @@ func runRoutes(args []string) { |
| if p.Host != r.Host { |
| continue |
| } |
| - fmt.Fprintf(w, " /%s\tpath\t%s\t—\t—\t—\t—\n", p.Prefix, dashIfEmpty(p.Target)) |
| + fmt.Fprintf(w, " /%s\tpath\t%s\t—\t—\t—\t—\t—\n", p.Prefix, dashIfEmpty(p.Target)) |
| } |
| } |
| w.Flush() |
| @@ -101,6 +111,18 @@ func runRoutes(args []string) { |
| mutedf(stdout, "(routes on this deployment's own box; other boxes in a fleet are not listed)") |
| } |
| +// replicationCell renders the replication rung. The empty column is named |
| +// rather than dashed: it is the default every route was on before the ladder |
| +// had a door, it behaves exactly like "stream", and a reader deciding |
| +// whether a large low-write database is costing more than it should needs to |
| +// see that stated, not inferred from a blank. |
| +func replicationCell(class string) string { |
| + if class == "" { |
| + return "default" |
| + } |
| + return class |
| +} |
| + |
| // dashIfEmpty renders an empty cell as an em dash, so a column that is |
| // legitimately blank reads as an answer rather than as missing data. |
| func dashIfEmpty(s string) string { |