carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package console
2
3// The narrowed restart door (apirestartscope.go, discussion #16). What is
4// pinned here, in the order it would hurt if lost: a selector that names
5// nothing is REFUSED rather than widened into an app-wide restart; the request
6// that does land goes to the log old boxes never read, leaving the app-wide
7// object untouched; a dry run writes and announces nothing; and the answer
8// names the routes it addressed, which is the whole complaint the flags exist
9// to answer.
10
11import (
12 "context"
13 "encoding/json"
14 "net/http"
15 "slices"
16 "strings"
17 "testing"
18
19 "github.com/carlosframework/platform/internal/registry"
20 "github.com/carlosframework/platform/internal/release"
21 "github.com/carlosframework/platform/internal/svcrole"
22)
23
24func scopeURL(fx *consoleFixture, app string) string {
25 return fx.ts.URL + "/api/cli/apps/bab/" + app + "/restart/scope"
26}
27
28// scopeFixture is a console whose registry holds a small titogo: two routes on
29// the relay bundle, one on router, one bound to nothing, one asleep, plus a
30// neighbour app and another account's row to prove the tenant boundary.
31func scopeFixture(t *testing.T, code string) (*consoleFixture, string) {
32 t.Helper()
33 fx, token := restartFixture(t, code)
34 reg, err := registry.Open(fx.regPath)
35 if err != nil {
36 t.Fatal(err)
37 }
38 for _, rt := range []registry.Route{
39 {Host: "connect.go.tito.io", Kind: "instance", Status: "ready", App: "titogo",
40 Account: "bab", Addr: "127.0.0.1:9001", Environment: "relay"},
41 {Host: "relay2.go.tito.io", Kind: "instance", Status: "ready", App: "titogo",
42 Account: "bab", Addr: "127.0.0.1:9002", Environment: "relay", Unit: "relay2.service"},
43 {Host: "platform.ie.go.tito.io", Kind: "instance", Status: "ready", App: "titogo",
44 Account: "bab", Addr: "127.0.0.1:9003", Environment: "router"},
45 {Host: "bill.go.tito.io", Kind: "instance", Status: "ready", App: "titogo",
46 Account: "bab", Addr: "127.0.0.1:9004"},
47 {Host: "sleepy.go.tito.io", Kind: "instance", Status: "ready", App: "titogo",
48 Account: "bab", Addr: "127.0.0.1:9005", Environment: "relay", Hibernate: true},
49 {Host: "neighbour.example.com", Kind: "instance", Status: "ready", App: "kass",
50 Account: "bab", Addr: "127.0.0.1:9006", Environment: "relay"},
51 {Host: "foreign.example.com", Kind: "instance", Status: "ready", App: "titogo",
52 Account: "zzzz", Addr: "127.0.0.1:9007", Environment: "relay"},
53 } {
54 if err := reg.Upsert(rt); err != nil {
55 t.Fatal(err)
56 }
57 }
58 return fx, token
59}
60
61// scopeLog reads the narrowed request log every NEW box watches.
62func scopeLog(t *testing.T, fx *consoleFixture, app string) *release.ScopedRestartGeneration {
63 t.Helper()
64 g, found, err := release.LoadRestartScopes(context.Background(), fx.rel, release.OpsAccount, app)
65 if err != nil {
66 t.Fatalf("read narrowed request log: %v", err)
67 }
68 if !found {
69 return &release.ScopedRestartGeneration{}
70 }
71 return g
72}
73
74type scopeAnswer struct {
75 Account string `json:"account"`
76 App string `json:"app"`
77 Scope struct {
78 Kind string `json:"kind"`
79 Value string `json:"value"`
80 } `json:"scope"`
81 DryRun bool `json:"dry_run"`
82 Requested bool `json:"requested"`
83 Routes []struct {
84 Host string `json:"host"`
85 Environment string `json:"environment"`
86 Unit string `json:"unit"`
87 Hibernate bool `json:"hibernate"`
88 } `json:"routes"`
89}
90
91func postScope(t *testing.T, fx *consoleFixture, token, body string) (int, scopeAnswer, []byte) {
92 t.Helper()
93 status, out := postBody(t, scopeURL(fx, "titogo"), token, body)
94 var res scopeAnswer
95 if status == http.StatusOK {
96 if err := json.Unmarshal(out, &res); err != nil {
97 t.Fatalf("decode %s: %v", out, err)
98 }
99 }
100 return status, res, out
101}
102
103func scopeHosts(res scopeAnswer) []string {
104 out := make([]string, 0, len(res.Routes))
105 for _, r := range res.Routes {
106 out = append(out, r.Host)
107 }
108 return out
109}
110
111// TestAPIRestartScopeEnvironmentAddressesOnlyThatBundle is the fix: a request
112// for one environment names the routes reading it — this app's, this account's
113// — and nothing else. The neighbouring app on the same bundle name and the
114// other account's identically-bound route are both absent, and that is the
115// tenant boundary, not a coincidence of the fixture.
116func TestAPIRestartScopeEnvironmentAddressesOnlyThatBundle(t *testing.T) {
117 fx, token := scopeFixture(t, "code-scope-env")
118 status, res, out := postScope(t, fx, token, `{"environment":"relay"}`)
119 if status != http.StatusOK {
120 t.Fatalf("restart scope = %d %s, want 200", status, out)
121 }
122 if !res.Requested || res.Scope.Kind != "environment" || res.Scope.Value != "relay" {
123 t.Fatalf("answer = %+v, want a requested environment scope", res)
124 }
125 want := []string{"connect.go.tito.io", "relay2.go.tito.io", "sleepy.go.tito.io"}
126 if got := scopeHosts(res); !slices.Equal(got, want) {
127 t.Fatalf("addressed %v, want %v", got, want)
128 }
129 // The sleeping route is NAMED and flagged rather than filtered out: an
130 // operator has to be able to see that it is in the set and will not cycle
131 // now, which is correct rather than a failure.
132 var sleepySeen bool
133 for _, r := range res.Routes {
134 if r.Host == "sleepy.go.tito.io" {
135 sleepySeen = r.Hibernate
136 }
137 if r.Host == "relay2.go.tito.io" && r.Unit != "relay2.service" {
138 t.Fatalf("route %+v lost the unit that owns its process", r)
139 }
140 }
141 if !sleepySeen {
142 t.Fatal("the hibernating route came back unflagged — an operator would read it as one that just cycled")
143 }
144 g := scopeLog(t, fx, "titogo")
145 if len(g.Requests) != 1 || g.Requests[0].Scope.Value != "relay" {
146 t.Fatalf("request log = %+v, want the one narrowed request", g.Requests)
147 }
148 if got := serial(t, fx); got != 1 {
149 t.Fatalf("serial after a narrowed request = %d, want 1 (otherwise boxes wait out the hourly floor)", got)
150 }
151}
152
153func TestAPIRestartScopeRecordsMessageAndTarget(t *testing.T) {
154 fx, token := scopeFixture(t, "code-scope-message")
155 status, _, out := postScope(t, fx, token, `{"environment":"relay","message":" cycle the relay workers "}`)
156 if status != http.StatusOK {
157 t.Fatalf("restart scope = %d %s, want 200", status, out)
158 }
159 events, err := fx.st.Events(OpsAccountID, "titogo", 1)
160 if err != nil || len(events) != 1 {
161 t.Fatalf("Events = %+v, %v", events, err)
162 }
163 got := events[0]
164 if got.Action != "restart" || got.TargetType != "environment" || got.Target != "relay" ||
165 got.Message != "cycle the relay workers" || got.Detail != "requested a restart of environment relay (cli)" {
166 t.Fatalf("activity = %+v", got)
167 }
168}
169
170func TestAPIRestartScopeRejectsInvalidMessageBeforeWriting(t *testing.T) {
171 fx, token := scopeFixture(t, "code-scope-message-invalid")
172 status, _, out := postScope(t, fx, token, `{"host":"bill.go.tito.io","message":"first\nsecond"}`)
173 if status != http.StatusBadRequest {
174 t.Fatalf("restart scope = %d %s, want 400", status, out)
175 }
176 if len(scopeLog(t, fx, "titogo").Requests) != 0 || restartGenEtag(t, fx, "titogo") != "" {
177 t.Fatal("an invalid message still wrote a restart request")
178 }
179 if events, err := fx.st.Events(OpsAccountID, "titogo", 10); err != nil || len(events) != 0 {
180 t.Fatalf("activity after invalid message = %+v, %v", events, err)
181 }
182}
183
184// TestAPIRestartScopeLeavesTheAppWideObjectAlone is the additive property at
185// the door: a narrowed request must not move the object every OLD box watches,
186// or the fleet's un-upgraded boxes would each cycle the whole app.
187func TestAPIRestartScopeLeavesTheAppWideObjectAlone(t *testing.T) {
188 fx, token := scopeFixture(t, "code-scope-additive")
189 if status, _, out := postScope(t, fx, token, `{"host":"bill.go.tito.io"}`); status != http.StatusOK {
190 t.Fatalf("restart scope = %d %s, want 200", status, out)
191 }
192 if got := restartGenEtag(t, fx, "titogo"); got != "" {
193 t.Fatalf("the app-wide restart generation exists (%q) after a narrowed request — every old box would cycle everything", got)
194 }
195}
196
197// TestAPIRestartScopeRefusesAnUnknownEnvironment: the refusal that keeps a
198// typo from becoming the bug being fixed. It names what the app actually has,
199// because a refusal exists to let somebody spot their own mistake.
200func TestAPIRestartScopeRefusesAnUnknownEnvironment(t *testing.T) {
201 fx, token := scopeFixture(t, "code-scope-typo")
202 status, _, out := postScope(t, fx, token, `{"environment":"relayy"}`)
203 if status != http.StatusBadRequest {
204 t.Fatalf("a typo'd environment = %d %s, want 400", status, out)
205 }
206 body := string(out)
207 for _, want := range []string{"relayy", "relay", "router"} {
208 if !strings.Contains(body, want) {
209 t.Fatalf("refusal %s does not name %q — it must say what it looked for and what exists", body, want)
210 }
211 }
212 if len(scopeLog(t, fx, "titogo").Requests) != 0 || restartGenEtag(t, fx, "titogo") != "" {
213 t.Fatal("a refused selector still wrote a request")
214 }
215 if got := serial(t, fx); got != 0 {
216 t.Fatalf("serial = %d after a refusal, want it untouched", got)
217 }
218}
219
220// TestAPIRestartScopeRefusesAnUnknownHost: the same rule for the other flag.
221func TestAPIRestartScopeRefusesAnUnknownHost(t *testing.T) {
222 fx, token := scopeFixture(t, "code-scope-host-typo")
223 status, _, out := postScope(t, fx, token, `{"host":"bil.go.tito.io"}`)
224 if status != http.StatusBadRequest {
225 t.Fatalf("an unknown host = %d %s, want 400", status, out)
226 }
227 if !strings.Contains(string(out), "bill.go.tito.io") {
228 t.Fatalf("refusal %s does not list the hosts this app has", out)
229 }
230 // Another account's route is not a host this caller may name, and the
231 // refusal must not leak that it exists.
232 if strings.Contains(string(out), "foreign.example.com") {
233 t.Fatalf("refusal %s named another account's route", out)
234 }
235}
236
237// TestAPIRestartScopeRefusesAMismatchedPair is the precedence rule's teeth.
238// --host wins over --environment, but only after the host is checked against
239// it: quietly ignoring the wider flag would hide that one of the two is a
240// typo, and this is not a verb to guess on.
241func TestAPIRestartScopeRefusesAMismatchedPair(t *testing.T) {
242 fx, token := scopeFixture(t, "code-scope-mismatch")
243 status, _, out := postScope(t, fx, token, `{"host":"platform.ie.go.tito.io","environment":"relay"}`)
244 if status != http.StatusBadRequest {
245 t.Fatalf("a mismatched pair = %d %s, want 400", status, out)
246 }
247 if !strings.Contains(string(out), "router") || !strings.Contains(string(out), "relay") {
248 t.Fatalf("refusal %s must name what the host actually reads and what was asked for", out)
249 }
250 if len(scopeLog(t, fx, "titogo").Requests) != 0 {
251 t.Fatal("a mismatched pair still wrote a request")
252 }
253}
254
255// TestAPIRestartScopeHostWinsOverAMatchingEnvironment: the pair is accepted
256// when it agrees, and the narrower flag is what lands.
257func TestAPIRestartScopeHostWinsOverAMatchingEnvironment(t *testing.T) {
258 fx, token := scopeFixture(t, "code-scope-pair")
259 status, res, out := postScope(t, fx, token, `{"host":"connect.go.tito.io","environment":"relay"}`)
260 if status != http.StatusOK {
261 t.Fatalf("a matching pair = %d %s, want 200", status, out)
262 }
263 if res.Scope.Kind != "host" || res.Scope.Value != "connect.go.tito.io" {
264 t.Fatalf("scope = %+v, want the host to win", res.Scope)
265 }
266 if got := scopeHosts(res); !slices.Equal(got, []string{"connect.go.tito.io"}) {
267 t.Fatalf("addressed %v, want the one route", got)
268 }
269}
270
271// TestAPIRestartScopeDryRunTouchesNothing: the answer an operator wanted
272// before cycling anything — the set, and no restart.
273func TestAPIRestartScopeDryRunTouchesNothing(t *testing.T) {
274 fx, token := scopeFixture(t, "code-scope-dry")
275 status, res, out := postScope(t, fx, token, `{"environment":"relay","dry_run":true}`)
276 if status != http.StatusOK {
277 t.Fatalf("dry run = %d %s, want 200", status, out)
278 }
279 if res.Requested || !res.DryRun {
280 t.Fatalf("answer = %+v, want dry_run true and requested false", res)
281 }
282 if got := scopeHosts(res); len(got) != 3 {
283 t.Fatalf("dry run addressed %v, want the same three routes the real request would", got)
284 }
285 if len(scopeLog(t, fx, "titogo").Requests) != 0 || restartGenEtag(t, fx, "titogo") != "" {
286 t.Fatal("a dry run wrote a request")
287 }
288 if got := serial(t, fx); got != 0 {
289 t.Fatalf("serial = %d after a dry run, want it untouched — a dry run must not even wake the fleet", got)
290 }
291}
292
293// TestAPIRestartScopeDryRunOfTheWholeApp: with no selector, the dry run is how
294// somebody sees what a plain `carlos restart --app` would cycle. That is the
295// blast radius the reporter said they would not run the verb to discover.
296func TestAPIRestartScopeDryRunOfTheWholeApp(t *testing.T) {
297 fx, token := scopeFixture(t, "code-scope-dry-app")
298 status, res, out := postScope(t, fx, token, `{"dry_run":true}`)
299 if status != http.StatusOK {
300 t.Fatalf("app-wide dry run = %d %s, want 200", status, out)
301 }
302 if res.Scope.Kind != "app" {
303 t.Fatalf("scope = %+v, want the whole app", res.Scope)
304 }
305 want := []string{"bill.go.tito.io", "connect.go.tito.io", "platform.ie.go.tito.io", "relay2.go.tito.io", "sleepy.go.tito.io"}
306 if got := scopeHosts(res); !slices.Equal(got, want) {
307 t.Fatalf("addressed %v, want every route of the app: %v", got, want)
308 }
309}
310
311// TestAPIRestartScopeRefusesAWriteWithNoSelector is the door's own
312// never-widen rule, turned on itself.
313//
314// This path exists BECAUSE the app-wide door widens a request it cannot read.
315// A body with no selector in it — a bare `{}`, or a client that renamed a
316// field — resolves to the whole app, and performing that here would be the
317// nineteen-route accident committed through the door built to prevent it. So
318// the app-wide case is answered only as a --dry-run (which is how somebody
319// sees the blast radius before typing the real verb), and a WRITE with no
320// narrowing is refused, naming the door that does own it.
321func TestAPIRestartScopeRefusesAWriteWithNoSelector(t *testing.T) {
322 fx, token := scopeFixture(t, "code-scope-appwide")
323 status, _, out := postScope(t, fx, token, `{}`)
324 if status != http.StatusBadRequest {
325 t.Fatalf("an app-wide write through the scope door = %d %s, want 400", status, out)
326 }
327 if !strings.Contains(string(out), "carlos restart --app") {
328 t.Fatalf("refusal %s does not name the door that cycles the whole app", out)
329 }
330 if got := restartGenEtag(t, fx, "titogo"); got != "" {
331 t.Fatalf("a refused app-wide write still moved the object every box watches (%q) — that is the whole app cycled", got)
332 }
333 if len(scopeLog(t, fx, "titogo").Requests) != 0 {
334 t.Fatal("a refused app-wide write still appended to the narrowed log")
335 }
336 if got := serial(t, fx); got != 0 {
337 t.Fatalf("serial = %d after a refusal, want it untouched", got)
338 }
339 // The dry run of the same body is still answered: that is the one form of
340 // app-wide this door is for.
341 if status, res, out := postScope(t, fx, token, `{"dry_run":true}`); status != http.StatusOK || res.Requested {
342 t.Fatalf("app-wide dry run = %d %s (%+v), want 200 and nothing requested", status, out, res)
343 }
344}
345
346// TestAPIRestartScopeRefusesAFieldItDoesNotKnow is the same rule one layer
347// out. A misspelled or renamed selector key that was merely IGNORED would
348// leave a body with no selector in it, which is a whole-app restart requested
349// by a client that asked for one route. The refusal names the field, because
350// the fix is a spelling.
351func TestAPIRestartScopeRefusesAFieldItDoesNotKnow(t *testing.T) {
352 fx, token := scopeFixture(t, "code-scope-unknown-field")
353 status, _, out := postScope(t, fx, token, `{"hosts":"relay2.go.tito.io"}`)
354 if status != http.StatusBadRequest {
355 t.Fatalf(`{"hosts":...} = %d %s, want 400 — an ignored key here is an app-wide restart`, status, out)
356 }
357 if !strings.Contains(string(out), "hosts") {
358 t.Fatalf("refusal %s does not name the field, so nobody can find the typo", out)
359 }
360 if restartGenEtag(t, fx, "titogo") != "" || len(scopeLog(t, fx, "titogo").Requests) != 0 {
361 t.Fatal("a body this door refused still wrote a request")
362 }
363}
364
365// TestAPIRestartScopeGateIsTheFamilys404: an app the account never claimed, a
366// stranger's token and a credential bound to another app are ONE answer, byte
367// for byte the answer the app-wide door gives — this repo's rule for these
368// gates is byte-identical, not merely same-status, because a differently
369// shaped 404 is an oracle that says "this door exists for you, that app does
370// not".
371func TestAPIRestartScopeGateIsTheFamilys404(t *testing.T) {
372 fx, _ := scopeFixture(t, "code-scope-gate")
373 // The old door's refusal, read from the old door rather than spelled out
374 // here, so the two cannot drift apart without this failing.
375 eve := cliToken(t, fx, "eve@keymail.dev", "code-scope-gate-eve")
376 oldStatus, oldBody := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/restart", eve, "")
377 if oldStatus != http.StatusNotFound {
378 t.Fatalf("the app-wide door answered a stranger %d, want 404", oldStatus)
379 }
380 for _, c := range []struct {
381 name string
382 url string
383 token string
384 }{
385 {"a stranger", scopeURL(fx, "titogo"), eve},
386 // An app-scoped service credential reaching PAST its own app. The
387 // binding is checked before the app is even looked up, so "not your
388 // app" and "no such app" stay the same answer.
389 {"a credential bound to another app", scopeURL(fx, "titogo"),
390 serviceToken(t, fx.st, OpsAccountID, "kassbot", "kass", svcrole.Operate)},
391 {"an app this account never claimed", scopeURL(fx, "nosuchapp"),
392 cliToken(t, fx, "alice@keymail.dev", "code-scope-gate-alice")},
393 } {
394 status, body := postBody(t, c.url, c.token, `{"environment":"relay"}`)
395 if status != oldStatus || string(body) != string(oldBody) {
396 t.Fatalf("%s got %d %q, want the app-wide door's own %d %q",
397 c.name, status, body, oldStatus, oldBody)
398 }
399 }
400 if len(scopeLog(t, fx, "titogo").Requests) != 0 || restartGenEtag(t, fx, "titogo") != "" {
401 t.Fatal("a refused caller still wrote a request")
402 }
403 if got := serial(t, fx); got != 0 {
404 t.Fatalf("serial = %d after refusals, want it untouched", got)
405 }
406}
407
408// TestAPIRestartScopeServiceCredentialReachesItsOwnApp is the other half of
409// the case above: the 404 there is about the app binding, not about services
410// being turned away from this door. An Operate credential bound to titogo
411// narrows a restart of titogo exactly as a member does.
412func TestAPIRestartScopeServiceCredentialReachesItsOwnApp(t *testing.T) {
413 fx, _ := scopeFixture(t, "code-scope-svc-own")
414 tok := serviceToken(t, fx.st, OpsAccountID, "titogobot", "titogo", svcrole.Operate)
415 status, res, out := postScope(t, fx, tok, `{"host":"bill.go.tito.io"}`)
416 if status != http.StatusOK {
417 t.Fatalf("a credential bound to titogo = %d %s, want 200", status, out)
418 }
419 if !res.Requested || res.Scope.Kind != "host" || res.Scope.Value != "bill.go.tito.io" {
420 t.Fatalf("answer = %+v, want the narrowed request", res)
421 }
422}
423
424// TestAPIRestartScopeNeedsARegistry: a console that cannot see routes cannot
425// resolve a selector or report a set, and must not answer "requested" for
426// something it never checked. The app-wide door still works there, and the
427// refusal says so.
428func TestAPIRestartScopeNeedsARegistry(t *testing.T) {
429 fx, token := scopeFixture(t, "code-scope-noreg")
430 fx.srv.cfg.Registry = ""
431 status, _, out := postScope(t, fx, token, `{"environment":"relay"}`)
432 if status != http.StatusServiceUnavailable {
433 t.Fatalf("a registry-less console answered %d %s, want 503", status, out)
434 }
435 if !strings.Contains(string(out), "carlos restart --app") {
436 t.Fatalf("refusal %s does not name the form that still works", out)
437 }
438}
439
440// TestAPIRestartScopeRefusesAMalformedSelector keeps a hostile value out
441// before the registry is even opened.
442func TestAPIRestartScopeRefusesAMalformedSelector(t *testing.T) {
443 fx, token := scopeFixture(t, "code-scope-malformed")
444 for _, body := range []string{
445 `{"host":"../../etc/passwd"}`,
446 `{"environment":"NOT VALID"}`,
447 `{"environment":"default"}`,
448 } {
449 if status, _, out := postScope(t, fx, token, body); status != http.StatusBadRequest {
450 t.Fatalf("%s = %d %s, want 400", body, status, out)
451 }
452 }
453}
454