| new file mode 100644 |
| index 000000000..0a0dac869 |
| --- /dev/null |
| +++ b/internal/edge/importgate_test.go |
| @@ -0,0 +1,380 @@ |
| +package edge |
| + |
| +import ( |
| + "net/http" |
| + "net/http/httptest" |
| + "strings" |
| + "sync" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/seeds" |
| +) |
| + |
| +// The import gate (seeded-instances spec §5.4). Neutral fixtures: app shop, |
| +// hosts under example.com. |
| +const ( |
| + gateHost = "old.example.com" |
| + gateAliasHost = "www.example.com" |
| + gateSeedID = "sd_aaaaaaaaaaaaaaaaaaaaaaaaaa" |
| + gateRefusal = "this instance is being imported. Try again in a few seconds.\n" |
| +) |
| + |
| +// gateBackend is an upstream that counts its requests and keeps the verify |
| +// header of the last one, so a test can say whether the app was reached and |
| +// whether the token reached it. |
| +type gateBackend struct { |
| + mu sync.Mutex |
| + n int |
| + sawVerify []string |
| +} |
| + |
| +func newGateBackend(t *testing.T, body, cacheControl string) (*httptest.Server, *gateBackend) { |
| + t.Helper() |
| + b := &gateBackend{} |
| + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + b.mu.Lock() |
| + b.n++ |
| + b.sawVerify = append(b.sawVerify, r.Header.Values(ImportVerifyHeader)...) |
| + b.mu.Unlock() |
| + if cacheControl != "" { |
| + w.Header().Set("Cache-Control", cacheControl) |
| + } |
| + w.Write([]byte(body)) |
| + })) |
| + t.Cleanup(srv.Close) |
| + return srv, b |
| +} |
| + |
| +func (b *gateBackend) seen() (int, []string) { |
| + b.mu.Lock() |
| + defer b.mu.Unlock() |
| + return b.n, append([]string(nil), b.sawVerify...) |
| +} |
| + |
| +// gateForImport swaps host's stored row for a gated import row backed by |
| +// addr, the way the swap does (one journal transaction), and returns the |
| +// row it wrote. The import row keeps the stored row's account and app. |
| +func gateForImport(t *testing.T, reg *registry.Registry, host, addr, gate string, hibernate bool) registry.Route { |
| + t.Helper() |
| + stored, err := reg.Get(host) |
| + if err != nil || stored == nil { |
| + t.Fatalf("get %s: %v %v", host, stored, err) |
| + } |
| + next := registry.Route{Host: host, Kind: "instance", Addr: addr, Status: "ready", App: stored.App, |
| + Account: stored.Account, Hibernate: hibernate, Provisioned: true, ImportGate: gate, |
| + SeedSource: `{"id":"` + gateSeedID + `","kind":"import","state":"swapped"}`} |
| + j := registry.ImportJournal{Host: host, SeedID: gateSeedID, Original: *stored, At: 1, |
| + RecordKey: "apps/bbw/shop/instances/" + host + ".json"} |
| + if err := reg.SwapForImport(j, next); err != nil { |
| + t.Fatalf("swap %s: %v", host, err) |
| + } |
| + return next |
| +} |
| + |
| +// gateScene is an edge with a legacy row at gateHost (and an alias of it), |
| +// swapped for a gated import row whose instance hibernates behind a fake |
| +// activator. |
| +type gateScene struct { |
| + reg *registry.Registry |
| + e *Edge |
| + act *fakeActivator |
| + legacy *gateBackend |
| + imported *gateBackend |
| + token string |
| +} |
| + |
| +func newGateScene(t *testing.T, importCacheControl string) *gateScene { |
| + t.Helper() |
| + legacySrv, legacy := newGateBackend(t, "legacy", "public, max-age=60") |
| + importSrv, imported := newGateBackend(t, "imported", importCacheControl) |
| + reg := openReg(t) |
| + if err := reg.Upsert(registry.Route{Host: gateHost, Kind: "service", Addr: legacySrv.Listener.Addr().String(), |
| + Status: "ready", App: "shop", Account: "bbw"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + upsertAlias(t, reg, gateAliasHost, gateHost) |
| + token, hash, err := seeds.NewVerifyToken() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + act := &fakeActivator{} |
| + e, err := New(reg, act) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(e.probers.Stop) |
| + e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)) |
| + // Before the swap the legacy app answers and its response is cached. |
| + if rec := gateDo(e, http.MethodGet, gateHost, "/up", ""); rec.Code != http.StatusOK || rec.Body.String() != "legacy" { |
| + t.Fatalf("before the swap: %d %q", rec.Code, rec.Body.String()) |
| + } |
| + gateForImport(t, reg, gateHost, importSrv.Listener.Addr().String(), hash, true) |
| + if err := e.Reload(); err != nil { |
| + t.Fatal(err) |
| + } |
| + return &gateScene{reg: reg, e: e, act: act, legacy: legacy, imported: imported, token: token} |
| +} |
| + |
| +func gateDo(h http.Handler, method, host, target, token string) *httptest.ResponseRecorder { |
| + req := httptest.NewRequest(method, "http://"+host+target, nil) |
| + req.Host = host |
| + if token != "" { |
| + req.Header.Set(ImportVerifyHeader, token) |
| + } |
| + rec := httptest.NewRecorder() |
| + h.ServeHTTP(rec, req) |
| + return rec |
| +} |
| + |
| +// wantGateRefusal is the gate's answer: 503, Retry-After 5, no-store, the |
| +// deck's sentence, and nothing of the token. |
| +func wantGateRefusal(t *testing.T, rec *httptest.ResponseRecorder, token, what string) { |
| + t.Helper() |
| + if rec.Code != http.StatusServiceUnavailable || rec.Header().Get("Retry-After") != "5" || |
| + rec.Header().Get("Cache-Control") != "no-store" || rec.Body.String() != gateRefusal { |
| + t.Fatalf("%s: %d retry %q cache %q body %q, want the gate's 503", what, rec.Code, |
| + rec.Header().Get("Retry-After"), rec.Header().Get("Cache-Control"), rec.Body.String()) |
| + } |
| + if token != "" && strings.Contains(rec.Body.String()+strings.Join(headerValues(rec.Header()), " "), token) { |
| + t.Fatalf("%s: the refusal echoed the token", what) |
| + } |
| +} |
| + |
| +func headerValues(h http.Header) []string { |
| + var out []string |
| + for k, vs := range h { |
| + out = append(out, k) |
| + out = append(out, vs...) |
| + } |
| + return out |
| +} |
| + |
| +// Every ordinary request to a gated row is refused before the cache and the |
| +// activator; only a GET carrying the right token reaches the instance, and |
| +// the app never sees the header. A closed gate refuses even the token, and |
| +// commit opens the row to everyone. |
| +func TestImportGateAtTheEdge(t *testing.T) { |
| + s := newGateScene(t, "") |
| + wrong, _, err := seeds.NewVerifyToken() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + refused := []struct { |
| + name, method, token string |
| + }{ |
| + {"no token", http.MethodGet, ""}, |
| + {"a wrong token", http.MethodGet, wrong}, |
| + {"the hash as the token", http.MethodGet, seeds.HashToken(s.token)}, |
| + {"a POST with the token", http.MethodPost, s.token}, |
| + {"a HEAD with the token", http.MethodHead, s.token}, |
| + {"a PUT with the token", http.MethodPut, s.token}, |
| + } |
| + for _, c := range refused { |
| + rec := gateDo(s.e, c.method, gateHost, "/up", c.token) |
| + if c.method == http.MethodHead { |
| + // A HEAD answer has no body to compare. |
| + if rec.Code != http.StatusServiceUnavailable || rec.Header().Get("Retry-After") != "5" { |
| + t.Fatalf("%s: %d, want the gate's 503", c.name, rec.Code) |
| + } |
| + continue |
| + } |
| + wantGateRefusal(t, rec, s.token, c.name) |
| + } |
| + if n, _ := s.imported.seen(); n != 0 || s.act.served != 0 { |
| + t.Fatalf("the gate let an ordinary request through: app %d, wakes %d", n, s.act.served) |
| + } |
| + if n, _ := s.legacy.seen(); n != 1 { |
| + t.Fatalf("the legacy app was reached %d times, want only the request before the swap", n) |
| + } |
| + |
| + // The verify request: GET with the token, on any path of the host. The |
| + // legacy response cached before the swap is not what it gets. |
| + rec := gateDo(s.e, http.MethodGet, gateHost, "/up", s.token) |
| + if rec.Code != http.StatusOK || rec.Body.String() != "imported" { |
| + t.Fatalf("verify = %d %q, want the imported app", rec.Code, rec.Body.String()) |
| + } |
| + if _, saw := s.imported.seen(); len(saw) != 0 { |
| + t.Fatalf("the app saw the verify header: %q", saw) |
| + } |
| + if s.act.served != 1 { |
| + t.Fatalf("the verify request woke the instance %d times, want 1", s.act.served) |
| + } |
| + |
| + // A rollback closes the gate: even the token is refused. |
| + if err := s.reg.CloseImportGate(gateHost, gateSeedID, 1); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := s.e.Reload(); err != nil { |
| + t.Fatal(err) |
| + } |
| + wantGateRefusal(t, gateDo(s.e, http.MethodGet, gateHost, "/up", s.token), s.token, "a closed gate") |
| + if n, _ := s.imported.seen(); n != 1 { |
| + t.Fatalf("a closed gate let the token through: app reached %d times", n) |
| + } |
| +} |
| + |
| +// Commit clears the gate: ordinary requests reach the instance, and a stray |
| +// verify header is still stripped before the app. |
| +func TestImportGateOpensOnCommit(t *testing.T) { |
| + s := newGateScene(t, "") |
| + if err := s.reg.CommitImport(gateHost, gateSeedID); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := s.e.Reload(); err != nil { |
| + t.Fatal(err) |
| + } |
| + if rec := gateDo(s.e, http.MethodPost, gateHost, "/orders", ""); rec.Code != http.StatusOK || rec.Body.String() != "imported" { |
| + t.Fatalf("after commit = %d %q, want the imported app", rec.Code, rec.Body.String()) |
| + } |
| + if rec := gateDo(s.e, http.MethodGet, gateHost, "/orders", s.token); rec.Code != http.StatusOK { |
| + t.Fatalf("after commit with a stray token = %d", rec.Code) |
| + } |
| + if _, saw := s.imported.seen(); len(saw) != 0 { |
| + t.Fatalf("the app saw the verify header after commit: %q", saw) |
| + } |
| +} |
| + |
| +// An alias of a gated host is the gated row under another name: refused |
| +// without the token, and the token admits a GET. |
| +func TestImportGateCoversAnAlias(t *testing.T) { |
| + s := newGateScene(t, "") |
| + wantGateRefusal(t, gateDo(s.e, http.MethodGet, gateAliasHost, "/up", ""), s.token, "an alias with no token") |
| + if n, _ := s.imported.seen(); n != 0 || s.act.served != 0 { |
| + t.Fatalf("an alias reached a gated row: app %d, wakes %d", n, s.act.served) |
| + } |
| + if rec := gateDo(s.e, http.MethodGet, gateAliasHost, "/up", s.token); rec.Code != http.StatusOK || rec.Body.String() != "imported" { |
| + t.Fatalf("verify through an alias = %d %q", rec.Code, rec.Body.String()) |
| + } |
| +} |
| + |
| +// A path route under a gated host is gated too, and the token does not |
| +// open it: while the host is gated nothing but the verify request reaches |
| +// anything, and the verify request reaches only the gated row. The other |
| +// direction holds as well: an ungated host's prefix that selects a gated |
| +// row is refused, token or not, because a verify names the import's host. |
| +func TestImportGateCoversPathRoutes(t *testing.T) { |
| + s := newGateScene(t, "") |
| + targetSrv, target := newGateBackend(t, "prefix target", "") |
| + frontSrv, front := newGateBackend(t, "front", "") |
| + for _, rt := range []registry.Route{ |
| + {Host: "next.example.com", Kind: "service", Addr: targetSrv.Listener.Addr().String(), Status: "ready", App: "shop", Account: "bbw"}, |
| + {Host: "front.example.com", Kind: "service", Addr: frontSrv.Listener.Addr().String(), Status: "ready", App: "shop", Account: "bbw"}, |
| + } { |
| + if err := s.reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + upsertPath(t, s.reg, gateHost, "shop", "next.example.com") |
| + upsertPath(t, s.reg, "front.example.com", "old", gateHost) |
| + if err := s.e.Reload(); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + for _, tok := range []string{"", s.token} { |
| + for _, p := range []string{"/shop", "/shop/cart", "//shop/cart", "/x/../shop"} { |
| + wantGateRefusal(t, gateDo(s.e, http.MethodGet, gateHost, p, tok), s.token, "the gated host's prefix "+p) |
| + } |
| + wantGateRefusal(t, gateDo(s.e, http.MethodGet, gateAliasHost, "/shop", tok), s.token, "the prefix through the alias") |
| + wantGateRefusal(t, gateDo(s.e, http.MethodGet, "front.example.com", "/old/up", tok), s.token, "a prefix selecting the gated row") |
| + } |
| + if n, _ := target.seen(); n != 0 { |
| + t.Fatalf("a path route under a gated host was reached %d times", n) |
| + } |
| + if n, _ := s.imported.seen(); n != 0 || s.act.served != 0 { |
| + t.Fatalf("the gated row was reached through a prefix: app %d, wakes %d", n, s.act.served) |
| + } |
| + |
| + // The ungated front's own route is not the import's business. |
| + if rec := gateDo(s.e, http.MethodGet, "front.example.com", "/home", ""); rec.Code != http.StatusOK || rec.Body.String() != "front" { |
| + t.Fatalf("the ungated front = %d %q", rec.Code, rec.Body.String()) |
| + } |
| + if n, _ := front.seen(); n != 1 { |
| + t.Fatalf("front reached %d times", n) |
| + } |
| + // A verify on a path no prefix claims reaches the gated row. |
| + if rec := gateDo(s.e, http.MethodGet, gateHost, "/up", s.token); rec.Code != http.StatusOK || rec.Body.String() != "imported" { |
| + t.Fatalf("verify beside a prefix = %d %q", rec.Code, rec.Body.String()) |
| + } |
| + |
| + // After commit the prefix routes as before. |
| + if err := s.reg.CommitImport(gateHost, gateSeedID); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := s.e.Reload(); err != nil { |
| + t.Fatal(err) |
| + } |
| + if rec := gateDo(s.e, http.MethodGet, gateHost, "/shop/cart", ""); rec.Code != http.StatusOK || rec.Body.String() != "prefix target" { |
| + t.Fatalf("the prefix after commit = %d %q", rec.Code, rec.Body.String()) |
| + } |
| +} |
| + |
| +// The verify request never touches the shared cache: it does not read the |
| +// legacy app's cached answer, and its own answer is not stored, so a |
| +// rollback that puts the legacy route back never replays the import's |
| +// response to visitors. |
| +func TestImportGateVerifyBypassesTheCache(t *testing.T) { |
| + s := newGateScene(t, "public, max-age=60") |
| + for i := 1; i <= 2; i++ { |
| + if rec := gateDo(s.e, http.MethodGet, gateHost, "/up", s.token); rec.Code != http.StatusOK || rec.Body.String() != "imported" { |
| + t.Fatalf("verify %d = %d %q", i, rec.Code, rec.Body.String()) |
| + } |
| + if n, _ := s.imported.seen(); n != i { |
| + t.Fatalf("verify %d reached the app %d times: served from the cache", i, n) |
| + } |
| + } |
| + // The rollback restores the legacy route. |
| + if err := s.reg.CloseImportGate(gateHost, gateSeedID, 1); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := s.reg.RestoreImportRoute(gateHost, gateSeedID); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := s.e.Reload(); err != nil { |
| + t.Fatal(err) |
| + } |
| + if rec := gateDo(s.e, http.MethodGet, gateHost, "/up", ""); rec.Code != http.StatusOK || rec.Body.String() != "legacy" { |
| + t.Fatalf("after the rollback = %d %q, want the legacy answer", rec.Code, rec.Body.String()) |
| + } |
| +} |
| + |
| +// A gated host homed on another box is gated by that box's edge: the gate |
| +// column travels with the row, and the sending edge has no row for the host, |
| +// so it backhauls the request untouched and the home edge decides. Without |
| +// the token the client gets the home edge's 503; with it, the import answers |
| +// and never sees the header. |
| +func TestImportGateHoldsAcrossABackhaul(t *testing.T) { |
| + s := newBackhaulScene(t) |
| + s.converge(t) |
| + importSrv, imported := newGateBackend(t, "imported", "") |
| + token, hash, err := seeds.NewVerifyToken() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + gateForImport(t, s.a.edge.reg, e2eHomedHost, importSrv.Listener.Addr().String(), hash, false) |
| + if err := s.a.edge.Reload(); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, ok := s.b.edge.lookupUpstream(e2eHomedHost); ok { |
| + t.Fatal("test bug: the sending edge must not route the gated host") |
| + } |
| + url := "https://" + e2eHomedHost + "/up" |
| + home := s.a.calls.Load() |
| + resp, body := s.get(t, s.b, url, nil, true) |
| + if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") != "5" || body != gateRefusal { |
| + t.Fatalf("backhauled with no token = %d retry %q body %q, want the home edge's gate", resp.StatusCode, resp.Header.Get("Retry-After"), body) |
| + } |
| + if got := s.a.calls.Load() - home; got != 1 { |
| + t.Fatalf("the home edge saw %d request(s), want the one it refused", got) |
| + } |
| + if n, _ := imported.seen(); n != 0 { |
| + t.Fatalf("a backhauled request with no token reached the import %d times", n) |
| + } |
| + resp, body = s.get(t, s.b, url, map[string]string{ImportVerifyHeader: token}, true) |
| + if resp.StatusCode != http.StatusOK || body != "imported" { |
| + t.Fatalf("backhauled verify = %d %q, want the import", resp.StatusCode, body) |
| + } |
| + if n, saw := imported.seen(); n != 1 || len(saw) != 0 { |
| + t.Fatalf("backhauled verify: import reached %d times, header at the app %q", n, saw) |
| + } |
| +} |
amadan