| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "reflect" |
| 6 | "sync/atomic" |
| 7 | "testing" |
| 8 | "time" |
| 9 | ) |
| 10 | |
| 11 | // The adoption a real box performs is delegated: the sandboxed agent cannot |
| 12 | // write the version store, so the ROOT pass stages, repoints live/<host> and |
| 13 | // stamps the version — and cycles no exec child, because the children are the |
| 14 | // agent's. Nothing then told the agent the symlink had moved, and the awake |
| 15 | // child served the old build until the sweep's grace ran out (discussion #231: |
| 16 | // 110-200s per deploy on the flagship). ConvergeAdopted is the agent finishing |
| 17 | // the half it delegated: no grace, no sweep. |
| 18 | func TestConvergeAdoptedCyclesAStaleInstanceWithoutTheGrace(t *testing.T) { |
| 19 | t.Parallel() |
| 20 | a, b, _, _ := testActivator(t, time.Hour) // idle far away — only convergence can stop it |
| 21 | var ver atomic.Value |
| 22 | ver.Store("v1") |
| 23 | a.liveVer = func(string) (string, error) { return ver.Load().(string), nil } |
| 24 | b.execVerFn = func() string { return ver.Load().(string) } |
| 25 | host := "one.example.com" |
| 26 | |
| 27 | release, err := a.beginRequest(context.Background(), testRoute(host)) |
| 28 | if err != nil { |
| 29 | t.Fatalf("beginRequest: %v", err) |
| 30 | } |
| 31 | release() |
| 32 | if got := a.Drifted(); len(got) != 0 { |
| 33 | t.Fatalf("Drifted = %v with the symlink unmoved, want none", got) |
| 34 | } |
| 35 | |
| 36 | ver.Store("v2") // the root pass repoints under the running child |
| 37 | if got, want := a.Drifted(), map[string]string{host: "v2"}; !reflect.DeepEqual(got, want) { |
| 38 | t.Fatalf("Drifted = %v, want %v", got, want) |
| 39 | } |
| 40 | if !a.ConvergeAdopted(host, "v2") { |
| 41 | t.Fatal("ConvergeAdopted refused an awake instance serving v1 under a symlink that names v2") |
| 42 | } |
| 43 | // No clock advance and no sweep: the cycle is the call's own doing. |
| 44 | waitFor(t, "the adopted-version cycle", func() bool { return a.entryCount() == 0 }) |
| 45 | if !b.lastProc().isStopped() { |
| 46 | t.Fatal("stale instance's process was not stopped") |
| 47 | } |
| 48 | if !b.lastRep().isClosed() { |
| 49 | t.Fatal("the cycle skipped the final checkpoint / lease release") |
| 50 | } |
| 51 | |
| 52 | // The next request wakes it onto v2, and a converged child is left alone. |
| 53 | release, err = a.beginRequest(context.Background(), testRoute(host)) |
| 54 | if err != nil { |
| 55 | t.Fatalf("re-wake after the cycle: %v", err) |
| 56 | } |
| 57 | release() |
| 58 | if got := a.Drifted(); len(got) != 0 { |
| 59 | t.Fatalf("Drifted = %v after re-waking onto v2, want none", got) |
| 60 | } |
| 61 | if a.ConvergeAdopted(host, "v2") { |
| 62 | t.Fatal("ConvergeAdopted cycled a child that already runs the adopted version") |
| 63 | } |
| 64 | if a.entryCount() != 1 { |
| 65 | t.Fatal("a converged instance was stopped") |
| 66 | } |
| 67 | } |
| 68 | |
| 69 | // Every refusal leaves the instance exactly as it was. In particular it never |
| 70 | // WAKES anything: a sleeper picks the new build up at its own next wake. |
| 71 | func TestConvergeAdoptedRefusals(t *testing.T) { |
| 72 | a, b, _, _ := testActivator(t, time.Hour) |
| 73 | var ver atomic.Value |
| 74 | ver.Store("v1") |
| 75 | a.liveVer = func(string) (string, error) { return ver.Load().(string), nil } |
| 76 | b.execVerFn = func() string { return ver.Load().(string) } |
| 77 | host := "one.example.com" |
| 78 | |
| 79 | if a.ConvergeAdopted(host, "v2") { |
| 80 | t.Fatal("ConvergeAdopted acted on a host that is asleep") |
| 81 | } |
| 82 | if a.entryCount() != 0 { |
| 83 | t.Fatal("ConvergeAdopted woke a sleeping host") |
| 84 | } |
| 85 | |
| 86 | release, err := a.beginRequest(context.Background(), testRoute(host)) |
| 87 | if err != nil { |
| 88 | t.Fatalf("beginRequest: %v", err) |
| 89 | } |
| 90 | release() |
| 91 | ver.Store("v2") |
| 92 | |
| 93 | // The caller's version and the symlink must agree. A stamp that names |
| 94 | // something the symlink does not is an adoption still in flight (or |
| 95 | // already superseded), and cycling now would land on whichever build the |
| 96 | // symlink happens to name. |
| 97 | if a.ConvergeAdopted(host, "v3") { |
| 98 | t.Fatal("ConvergeAdopted cycled onto a version the live symlink does not name") |
| 99 | } |
| 100 | if a.ConvergeAdopted(host, "") { |
| 101 | t.Fatal("ConvergeAdopted cycled for an empty version") |
| 102 | } |
| 103 | if a.entryCount() != 1 { |
| 104 | t.Fatal("a refused ConvergeAdopted stopped the instance") |
| 105 | } |
| 106 | } |
| 107 | |
| 108 | // An unknown spawn version is "don't converge", the same rule the sweep keeps: |
| 109 | // with nothing to compare, a cycle could never be shown to have been needed. |
| 110 | func TestConvergeAdoptedLeavesAnUnknownSpawnVersionAlone(t *testing.T) { |
| 111 | a, b, _, _ := testActivator(t, time.Hour) |
| 112 | a.liveVer = func(string) (string, error) { return "v2", nil } |
| 113 | b.execVerFn = func() string { return "" } |
| 114 | host := "one.example.com" |
| 115 | |
| 116 | release, err := a.beginRequest(context.Background(), testRoute(host)) |
| 117 | if err != nil { |
| 118 | t.Fatalf("beginRequest: %v", err) |
| 119 | } |
| 120 | release() |
| 121 | if got := a.Drifted(); len(got) != 0 { |
| 122 | t.Fatalf("Drifted = %v for a child whose spawn version is unknown, want none", got) |
| 123 | } |
| 124 | if a.ConvergeAdopted(host, "v2") { |
| 125 | t.Fatal("ConvergeAdopted cycled a child whose spawn version is unknown") |
| 126 | } |
| 127 | } |
| 128 | |
| 129 | // A worker is out of this path's reach on purpose: its registry version is |
| 130 | // what the child REPORTS, not what adoption stamped, so "the stamp agrees with |
| 131 | // the symlink" means something else there. The sweep's own worker converge |
| 132 | // (one move, behind the grace) still covers it. |
| 133 | func TestConvergeAdoptedLeavesWorkersToTheSweep(t *testing.T) { |
| 134 | t.Parallel() |
| 135 | a, _, _, ver := shippingActivator(t, time.Hour) |
| 136 | host := "bot.x.internal" |
| 137 | if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| 138 | t.Fatalf("WakeWorker: %v", err) |
| 139 | } |
| 140 | waitFor(t, "the worker to be awake", func() bool { return a.entrySpawnVersion(host) == "v1" }) |
| 141 | ver.Store("v2") |
| 142 | if got := a.Drifted(); len(got) != 0 { |
| 143 | t.Fatalf("Drifted = %v, want no workers in it", got) |
| 144 | } |
| 145 | if a.ConvergeAdopted(host, "v2") { |
| 146 | t.Fatal("ConvergeAdopted cycled a worker") |
| 147 | } |
| 148 | } |
| 149 | |
| 150 | // An adopt pass's restart callback fires after it has repointed live/<host>, |
| 151 | // so by then the symlink names the version being adopted. A child ALREADY |
| 152 | // exec'd from that version has nothing to pick up: stopping it is one more |
| 153 | // drain and wake for no change of build. That child exists whenever something |
| 154 | // else got there first — ConvergeAdopted finishing a root pass's adoption and |
| 155 | // a request re-waking the instance, while a slower pass was still mid-host. |
| 156 | func TestBounceAdoptedSparesAChildAlreadyOnTheLiveBuild(t *testing.T) { |
| 157 | t.Parallel() |
| 158 | a, b, _, _ := testActivator(t, time.Hour) |
| 159 | var ver atomic.Value |
| 160 | ver.Store("v2") |
| 161 | a.liveVer = func(string) (string, error) { return ver.Load().(string), nil } |
| 162 | b.execVerFn = func() string { return ver.Load().(string) } |
| 163 | host := "one.example.com" |
| 164 | |
| 165 | release, err := a.beginRequest(context.Background(), testRoute(host)) |
| 166 | if err != nil { |
| 167 | t.Fatalf("beginRequest: %v", err) |
| 168 | } |
| 169 | release() |
| 170 | if a.BounceAdopted(host) { |
| 171 | t.Fatal("BounceAdopted stopped a child that already runs the build the symlink names") |
| 172 | } |
| 173 | if a.entryCount() != 1 { |
| 174 | t.Fatal("the converged child was stopped") |
| 175 | } |
| 176 | |
| 177 | // The ordinary case is unchanged: the symlink has moved under the child. |
| 178 | ver.Store("v3") |
| 179 | if !a.BounceAdopted(host) { |
| 180 | t.Fatal("BounceAdopted spared a child serving v2 under a symlink that names v3") |
| 181 | } |
| 182 | waitFor(t, "the adoption bounce", func() bool { return a.entryCount() == 0 }) |
| 183 | if a.BounceAdopted(host) { |
| 184 | t.Fatal("BounceAdopted acted on a host that is asleep") |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | // With nothing to compare it bounces, exactly as Bounce always has: a backend |
| 189 | // that cannot name its build, or a symlink that cannot be read, must not turn |
| 190 | // an adoption into a silent no-op over a child still on the old binary. |
| 191 | func TestBounceAdoptedBouncesWhenTheBuildIsUnknown(t *testing.T) { |
| 192 | t.Parallel() |
| 193 | a, b, _, _ := testActivator(t, time.Hour) |
| 194 | a.liveVer = func(string) (string, error) { return "v2", nil } |
| 195 | b.execVerFn = func() string { return "" } |
| 196 | host := "one.example.com" |
| 197 | |
| 198 | release, err := a.beginRequest(context.Background(), testRoute(host)) |
| 199 | if err != nil { |
| 200 | t.Fatalf("beginRequest: %v", err) |
| 201 | } |
| 202 | release() |
| 203 | if !a.BounceAdopted(host) { |
| 204 | t.Fatal("BounceAdopted refused a child whose spawn version is unknown") |
| 205 | } |
| 206 | waitFor(t, "the bounce", func() bool { return a.entryCount() == 0 }) |
| 207 | } |
| 208 | |
| 209 | // A bounce that lands while the instance is still waking is not dropped. The |
| 210 | // wake already read its config and execed its build, so whatever the bounce |
| 211 | // was for — new config, a `carlos restart` — has not reached the child, and |
| 212 | // nothing will ask again: config delivery bounces only when the rendered |
| 213 | // file changes. The bounce is held and spent the moment the wake finishes, |
| 214 | // before any waiting request is let through, so every request lands on a |
| 215 | // child started after the bounce was asked for. |
| 216 | func TestBounceDuringAWakeIsAppliedWhenTheWakeFinishes(t *testing.T) { |
| 217 | t.Parallel() |
| 218 | a, b, _, _ := testActivator(t, time.Hour) |
| 219 | b.startGate = make(chan struct{}) |
| 220 | host := "one.example.com" |
| 221 | |
| 222 | served := make(chan error, 1) |
| 223 | go func() { |
| 224 | release, err := a.beginRequest(context.Background(), testRoute(host)) |
| 225 | if err == nil { |
| 226 | release() |
| 227 | } |
| 228 | served <- err |
| 229 | }() |
| 230 | waitFor(t, "the wake to reach Start", func() bool { return b.startCount() == 1 }) |
| 231 | |
| 232 | if a.Bounce(host) { |
| 233 | t.Fatal("Bounce reported a stop on an instance that is still waking") |
| 234 | } |
| 235 | close(b.startGate) |
| 236 | |
| 237 | if err := <-served; err != nil { |
| 238 | t.Fatalf("the request waiting on the wake: %v", err) |
| 239 | } |
| 240 | // It was served by a SECOND child: the first was stopped as soon as its |
| 241 | // wake finished, and the waiting request woke a fresh one. |
| 242 | if got := b.startCount(); got != 2 { |
| 243 | t.Fatalf("children started = %d, want 2: the bounce asked for during the wake was dropped", got) |
| 244 | } |
| 245 | if !b.procs[0].isStopped() { |
| 246 | t.Fatal("the child from the wake the bounce landed in is still running") |
| 247 | } |
| 248 | if a.Bounce(host) != true { |
| 249 | t.Fatal("an ordinary bounce of the awake replacement was refused") |
| 250 | } |
| 251 | } |
| 252 | |
amadan