| 1 | package main |
| 2 | |
| 3 | // `carlos route --replication` — the door the replication ladder did not |
| 4 | // have. The ladder was built and honoured and unreachable: three named |
| 5 | // rungs, one of them reachable, and it was the most expensive one |
| 6 | // (discussion #10). |
| 7 | |
| 8 | import ( |
| 9 | "os/exec" |
| 10 | "path/filepath" |
| 11 | "strings" |
| 12 | "testing" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/registry" |
| 15 | ) |
| 16 | |
| 17 | // TestReplicationRailRefusesNonHibernatingRoutes pins the first of this |
| 18 | // verb's two refusals, as a pure function for the reason unitRail and |
| 19 | // backingRail are: the class is read in exactly one place — the activator's |
| 20 | // wake path — and /etc/litestream.yml covers everything the box-wide |
| 21 | // replicator is not told to leave alone, at its own interval. On a route that |
| 22 | // does not hibernate the column would be written, reported as set, and read |
| 23 | // by nothing, which is the same silent no-op the missing door was. |
| 24 | func TestReplicationRailRefusesNonHibernatingRoutes(t *testing.T) { |
| 25 | awake := registry.Route{ |
| 26 | Host: "console.carloku.com", Kind: "instance", App: "console", |
| 27 | Addr: "127.0.0.1:8090", DBPath: "/data/console.carloku.com.db", |
| 28 | } |
| 29 | err := replicationRail(awake) |
| 30 | if err == nil { |
| 31 | t.Fatal("replicationRail(non-hibernating route) accepted; want a refusal") |
| 32 | } |
| 33 | for _, want := range []string{"hibernate", "litestream"} { |
| 34 | if !strings.Contains(err.Error(), want) { |
| 35 | t.Errorf("refusal %q does not mention %q — the operator has to be told which replicator owns this database instead", err, want) |
| 36 | } |
| 37 | } |
| 38 | |
| 39 | hibernating := awake |
| 40 | hibernating.Hibernate = true |
| 41 | if err := replicationRail(hibernating); err != nil { |
| 42 | t.Fatalf("replicationRail(hibernating route) = %v, want accepted", err) |
| 43 | } |
| 44 | } |
| 45 | |
| 46 | // TestReplicationRailRefusesAHibernatingRouteWithNoDatabase pins the second |
| 47 | // half of the same premise, which the first version of this rail missed. |
| 48 | // Box-wide litestream excludes a route on `r.Hibernate && r.DBPath != ""` |
| 49 | // (ops.FleetDBsWithSkipped) — hibernation ALONE excludes nothing. A |
| 50 | // hibernating route whose row names no database is claimed by the dataDir |
| 51 | // glob like any other file and replicated at /etc/litestream.yml's interval, |
| 52 | // so a rung set on it would be written, reported by `carlos routes`, and |
| 53 | // honoured by nobody: the exact failure the rail exists to refuse, reached |
| 54 | // through the rail. |
| 55 | // |
| 56 | // Not hypothetical: `carlos add` defaults db_path only for a single-upstream |
| 57 | // non-unit `instance`, so a static or service route, or an instance record |
| 58 | // carrying an upstream set (provision.Record.DerivesPaths is false there), |
| 59 | // hibernates with the column empty. |
| 60 | func TestReplicationRailRefusesAHibernatingRouteWithNoDatabase(t *testing.T) { |
| 61 | pathless := registry.Route{ |
| 62 | Host: "static.example.com", Kind: "static", App: "site", |
| 63 | Hibernate: true, |
| 64 | } |
| 65 | err := replicationRail(pathless) |
| 66 | if err == nil { |
| 67 | t.Fatal("replicationRail(hibernating route with no db_path) accepted; want a refusal") |
| 68 | } |
| 69 | for _, want := range []string{"db_path", "--db"} { |
| 70 | if !strings.Contains(err.Error(), want) { |
| 71 | t.Errorf("refusal %q does not mention %q — the operator has to be told what is missing and which door supplies it", err, want) |
| 72 | } |
| 73 | } |
| 74 | } |
| 75 | |
| 76 | // TestReplicationWordNamesTheDefault: "" is this column's value on every |
| 77 | // route that predates the door, it behaves exactly like stream, and printing |
| 78 | // it as an empty string reads as a bug rather than as the answer it is. |
| 79 | func TestReplicationWordNamesTheDefault(t *testing.T) { |
| 80 | if got := replicationWord(""); !strings.Contains(got, "default") { |
| 81 | t.Errorf(`replicationWord("") = %q, want it to name the default`, got) |
| 82 | } |
| 83 | for _, class := range registry.ReplicationClasses() { |
| 84 | got := replicationWord(class) |
| 85 | if !strings.Contains(got, class) { |
| 86 | t.Errorf("replicationWord(%q) = %q, want it to name the rung", class, got) |
| 87 | } |
| 88 | if !strings.Contains(got, "loss window") { |
| 89 | t.Errorf("replicationWord(%q) = %q, want the loss window spelled out — that is the whole trade", class, got) |
| 90 | } |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | // TestReplicationCellNamesTheDefault: the console-side listing has the same |
| 95 | // duty. Unlike ENVIRONMENT, whose empty value is dashed, this one is named: |
| 96 | // it is the column where "no answer" cost one tenant 97% of a day's replica |
| 97 | // traffic. |
| 98 | func TestReplicationCellNamesTheDefault(t *testing.T) { |
| 99 | if got := replicationCell(""); got != "default" { |
| 100 | t.Errorf(`replicationCell("") = %q, want "default"`, got) |
| 101 | } |
| 102 | if got := replicationCell(registry.ReplicationDaily); got != "daily" { |
| 103 | t.Errorf("replicationCell(daily) = %q", got) |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | // TestRouteReplicationSetsAndClearsTheRung drives the whole door as a |
| 108 | // subprocess, the shape it ships in: the routes that need a cheaper rung |
| 109 | // already exist on live boxes, so this stamps a LIVE registry row rather than |
| 110 | // needing the route recreated. |
| 111 | // |
| 112 | // It pins the failure a careless door would cause as much as the one the |
| 113 | // missing door did: replicationSyncInterval answers an unknown class with |
| 114 | // litestream's default, so an accepted typo is a route that quietly never |
| 115 | // left the most expensive rung. |
| 116 | func TestRouteReplicationSetsAndClearsTheRung(t *testing.T) { |
| 117 | bin := buildCarlos(t) |
| 118 | tmp := t.TempDir() |
| 119 | scratch := t.TempDir() |
| 120 | regPath := filepath.Join(tmp, "registry.db") |
| 121 | |
| 122 | run := func(args ...string) (string, error) { |
| 123 | t.Helper() |
| 124 | cmd := exec.Command(bin, args...) |
| 125 | cmd.Env = carlosTestEnv(t) |
| 126 | // pick() walks cwd ancestors for .carlos/config, so the subprocess |
| 127 | // must not run from the real checkout. |
| 128 | cmd.Dir = scratch |
| 129 | out, err := cmd.CombinedOutput() |
| 130 | return string(out), err |
| 131 | } |
| 132 | replicationOf := func(host string) string { |
| 133 | t.Helper() |
| 134 | reg, err := registry.Open(regPath) |
| 135 | if err != nil { |
| 136 | t.Fatalf("open registry: %v", err) |
| 137 | } |
| 138 | defer reg.Close() |
| 139 | rt, err := reg.Get(host) |
| 140 | if err != nil || rt == nil { |
| 141 | t.Fatalf("get %s: %+v, %v", host, rt, err) |
| 142 | } |
| 143 | return rt.Replication |
| 144 | } |
| 145 | |
| 146 | // A hibernating instance as one exists on a box today: a large database, |
| 147 | // low write volume, on the every-second default because nothing could say |
| 148 | // otherwise. |
| 149 | if out, err := run("add", "-registry", regPath, "-app", "seapointish", |
| 150 | "-socket", filepath.Join(tmp, "s.sock"), "-hibernate", "seapointish.example.com"); err != nil { |
| 151 | t.Fatalf("add route: %v\n%s", err, out) |
| 152 | } |
| 153 | if got := replicationOf("seapointish.example.com"); got != "" { |
| 154 | t.Fatalf("a fresh route already carries a rung: %q", got) |
| 155 | } |
| 156 | |
| 157 | // A typo must cost nothing and reach nothing. |
| 158 | out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", "-replication", "dayly") |
| 159 | if err == nil { |
| 160 | t.Fatalf("route --replication dayly: want non-zero exit, got 0\n%s", out) |
| 161 | } |
| 162 | if got := replicationOf("seapointish.example.com"); got != "" { |
| 163 | t.Fatalf("a refused --replication still wrote %q", got) |
| 164 | } |
| 165 | |
| 166 | // A bare "" is refused rather than silently treated as a clear: the doors |
| 167 | // that clear say so out loud. |
| 168 | if out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", "-replication", ""); err == nil { |
| 169 | t.Fatalf("route --replication \"\": want non-zero exit, got 0\n%s", out) |
| 170 | } |
| 171 | |
| 172 | // -dry-run runs the rail and prints the plan, writing nothing. |
| 173 | out, err = run("route", "-registry", regPath, "-host", "seapointish.example.com", |
| 174 | "-replication", "daily", "-dry-run") |
| 175 | if err != nil { |
| 176 | t.Fatalf("route --replication --dry-run: %v\n%s", err, out) |
| 177 | } |
| 178 | if !strings.Contains(out, "daily") { |
| 179 | t.Errorf("dry run output = %q, want it to name the rung", out) |
| 180 | } |
| 181 | if got := replicationOf("seapointish.example.com"); got != "" { |
| 182 | t.Fatalf("--dry-run wrote %q", got) |
| 183 | } |
| 184 | |
| 185 | // The real write. |
| 186 | if out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", |
| 187 | "-replication", "daily"); err != nil { |
| 188 | t.Fatalf("route --replication daily: %v\n%s", err, out) |
| 189 | } |
| 190 | if got := replicationOf("seapointish.example.com"); got != registry.ReplicationDaily { |
| 191 | t.Fatalf("replication = %q, want %q", got, registry.ReplicationDaily) |
| 192 | } |
| 193 | |
| 194 | // And the way back, which is the sentinel rather than a name to type. |
| 195 | if out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", |
| 196 | "-replication", "-"); err != nil { |
| 197 | t.Fatalf("route --replication -: %v\n%s", err, out) |
| 198 | } |
| 199 | if got := replicationOf("seapointish.example.com"); got != "" { |
| 200 | t.Fatalf("--replication - left %q, want the default", got) |
| 201 | } |
| 202 | |
| 203 | // The listing has to answer the question the setter just decided, or an |
| 204 | // operator cannot see what they set — which was half of #10's complaint. |
| 205 | if out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", |
| 206 | "-replication", "batched"); err != nil { |
| 207 | t.Fatalf("route --replication batched: %v\n%s", err, out) |
| 208 | } |
| 209 | out, err = run("instances", "-registry", regPath) |
| 210 | if err != nil { |
| 211 | t.Fatalf("instances: %v\n%s", err, out) |
| 212 | } |
| 213 | if !strings.Contains(out, "REPL") || !strings.Contains(out, "batched") { |
| 214 | t.Errorf("`carlos instances` does not show the rung:\n%s", out) |
| 215 | } |
| 216 | } |
| 217 | |
| 218 | // TestRouteReplicationRefusesCombinationWithOtherAxes: the dispatch runs one |
| 219 | // arm, so a command naming two axes would write one and drop the other on the |
| 220 | // floor without a word — the silent partial --backing's exclusions were added |
| 221 | // for. --app is refused outright: a bulk durability change is precisely the |
| 222 | // operation nobody should be able to type by accident. |
| 223 | func TestRouteReplicationRefusesCombinationWithOtherAxes(t *testing.T) { |
| 224 | bin := buildCarlos(t) |
| 225 | tmp := t.TempDir() |
| 226 | scratch := t.TempDir() |
| 227 | regPath := filepath.Join(tmp, "registry.db") |
| 228 | |
| 229 | run := func(args ...string) (string, error) { |
| 230 | t.Helper() |
| 231 | cmd := exec.Command(bin, args...) |
| 232 | cmd.Env = carlosTestEnv(t) |
| 233 | cmd.Dir = scratch |
| 234 | out, err := cmd.CombinedOutput() |
| 235 | return string(out), err |
| 236 | } |
| 237 | if out, err := run("add", "-registry", regPath, "-app", "a", |
| 238 | "-socket", filepath.Join(tmp, "a.sock"), "-hibernate", "a.example.com"); err != nil { |
| 239 | t.Fatalf("add: %v\n%s", err, out) |
| 240 | } |
| 241 | for _, extra := range [][]string{ |
| 242 | {"-hibernate=false"}, |
| 243 | {"-channel", "edge"}, |
| 244 | {"-unit", "a.service"}, |
| 245 | } { |
| 246 | args := append([]string{"route", "-registry", regPath, "-host", "a.example.com", "-replication", "daily"}, extra...) |
| 247 | if out, err := run(args...); err == nil { |
| 248 | t.Errorf("route %v: want a refusal, got 0\n%s", extra, out) |
| 249 | } |
| 250 | } |
| 251 | if out, err := run("route", "-registry", regPath, "-app", "a", "-replication", "daily"); err == nil { |
| 252 | t.Errorf("route --app --replication: want a refusal, got 0\n%s", out) |
| 253 | } |
| 254 | } |
| 255 | |
| 256 | // TestRouteHibernateOffClearsTheRung: the rail is point-in-time, and until |
| 257 | // this landed nothing held it afterwards. A route set to `daily` could stop |
| 258 | // hibernating — by this verb, by `carlos add` with no --hibernate, or by a |
| 259 | // converge pass reading an instance record — and `carlos routes` went on |
| 260 | // printing `daily` while box-wide /etc/litestream.yml replicated the database |
| 261 | // at its own interval. A column that lies is the failure the door was built |
| 262 | // to end, so it must not survive the flip that makes it false. |
| 263 | // |
| 264 | // The clearing itself lives in registry.SetHibernate, where every door that |
| 265 | // turns hibernation off passes; what is pinned here is the operator-facing |
| 266 | // half — the flip works through the real binary, and it SAYS what it spent, |
| 267 | // because the rung is the one operator-typed value this command drops |
| 268 | // without being asked to. |
| 269 | func TestRouteHibernateOffClearsTheRung(t *testing.T) { |
| 270 | bin := buildCarlos(t) |
| 271 | tmp := t.TempDir() |
| 272 | scratch := t.TempDir() |
| 273 | regPath := filepath.Join(tmp, "registry.db") |
| 274 | |
| 275 | run := func(args ...string) (string, error) { |
| 276 | t.Helper() |
| 277 | cmd := exec.Command(bin, args...) |
| 278 | cmd.Env = carlosTestEnv(t) |
| 279 | cmd.Dir = scratch |
| 280 | out, err := cmd.CombinedOutput() |
| 281 | return string(out), err |
| 282 | } |
| 283 | rowOf := func(host string) (string, bool) { |
| 284 | t.Helper() |
| 285 | reg, err := registry.Open(regPath) |
| 286 | if err != nil { |
| 287 | t.Fatalf("open registry: %v", err) |
| 288 | } |
| 289 | defer reg.Close() |
| 290 | rt, err := reg.Get(host) |
| 291 | if err != nil || rt == nil { |
| 292 | t.Fatalf("get %s: %+v, %v", host, rt, err) |
| 293 | } |
| 294 | return rt.Replication, rt.Hibernate |
| 295 | } |
| 296 | |
| 297 | host := "seapointish.example.com" |
| 298 | if out, err := run("add", "-registry", regPath, "-app", "seapointish", |
| 299 | "-socket", filepath.Join(tmp, "s.sock"), "-hibernate", host); err != nil { |
| 300 | t.Fatalf("add route: %v\n%s", err, out) |
| 301 | } |
| 302 | if out, err := run("route", "-registry", regPath, "-host", host, "-replication", "daily"); err != nil { |
| 303 | t.Fatalf("route --replication daily: %v\n%s", err, out) |
| 304 | } |
| 305 | if class, _ := rowOf(host); class != registry.ReplicationDaily { |
| 306 | t.Fatalf("setup: replication = %q, want %q", class, registry.ReplicationDaily) |
| 307 | } |
| 308 | |
| 309 | out, err := run("route", "-registry", regPath, "-host", host, "-hibernate=false") |
| 310 | if err != nil { |
| 311 | t.Fatalf("route --hibernate=false: %v\n%s", err, out) |
| 312 | } |
| 313 | class, hib := rowOf(host) |
| 314 | if hib { |
| 315 | t.Fatalf("setup: the route still hibernates\n%s", out) |
| 316 | } |
| 317 | if class != "" { |
| 318 | t.Fatalf("replication = %q on a route that no longer hibernates — `carlos routes` would print a rung the box replicator does not honour", class) |
| 319 | } |
| 320 | // Told, not discovered. |
| 321 | for _, want := range []string{"replication", "cleared"} { |
| 322 | if !strings.Contains(out, want) { |
| 323 | t.Errorf("--hibernate=false output does not mention %q:\n%s", want, out) |
| 324 | } |
| 325 | } |
| 326 | |
| 327 | // And the door is shut behind it: the rung cannot be re-set while the |
| 328 | // route is awake, which is what makes the clear an honest state rather |
| 329 | // than a gap between two commands. |
| 330 | if out, err := run("route", "-registry", regPath, "-host", host, "-replication", "daily"); err == nil { |
| 331 | t.Errorf("route --replication on a woken route: want a refusal, got 0\n%s", out) |
| 332 | } |
| 333 | } |
| 334 | |
amadan