| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "net/http" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/provision" |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | "github.com/carlosframework/platform/internal/seeds" |
| 14 | ) |
| 15 | |
| 16 | const seedMoveHost = "canary.example.com" |
| 17 | |
| 18 | // seedMoveFixture is a console whose ops account runs the shop app with |
| 19 | // instances on, and a member's CLI token (an operator's when phase1, with |
| 20 | // the Phase 1 switch on). |
| 21 | func seedMoveFixture(t *testing.T, code string, phase1 bool) (*consoleFixture, string) { |
| 22 | t.Helper() |
| 23 | fx := newConsoleFixture(t) |
| 24 | now := time.Now() |
| 25 | if _, err := fx.st.CreateAppIn(OpsAccountID, "shop", "paul@keymail.dev", now); err != nil { |
| 26 | t.Fatal(err) |
| 27 | } |
| 28 | if err := fx.st.AddMember(OpsAccountID, "alice@keymail.dev", RoleMember, now); err != nil { |
| 29 | t.Fatal(err) |
| 30 | } |
| 31 | token := cliToken(t, fx, "alice@keymail.dev", code) |
| 32 | if phase1 { |
| 33 | if err := fx.st.SetOperator("alice@keymail.dev", true); err != nil { |
| 34 | t.Fatal(err) |
| 35 | } |
| 36 | enableMovesPhase1(fx) |
| 37 | } |
| 38 | enableInstances(t, fx, "shop", provision.Manifest{Schema: 1, App: "shop", Domain: "example.com", |
| 39 | Channel: "stable", Regions: []string{"ie"}}) |
| 40 | return fx, token |
| 41 | } |
| 42 | |
| 43 | // putSeededRecord creates an instance record of the shop app in region, |
| 44 | // holding a seed of kind in state. |
| 45 | func putSeededRecord(t *testing.T, st release.Store, host, region, kind, state string) { |
| 46 | t.Helper() |
| 47 | id, err := seeds.NewID() |
| 48 | if err != nil { |
| 49 | t.Fatal(err) |
| 50 | } |
| 51 | rec := provision.Record{Schema: 1, Host: host, Slug: strings.SplitN(host, ".", 2)[0], App: "shop", Account: release.OpsAccount, |
| 52 | Region: region, Channel: "stable", CreatedAt: time.Now().UTC(), |
| 53 | Seed: &provision.SeedSource{ID: id, Kind: kind, State: state, By: "paul@keymail.dev", At: time.Now().UTC()}} |
| 54 | if err := provision.CreateRecord(context.Background(), st, rec); err != nil { |
| 55 | t.Fatal(err) |
| 56 | } |
| 57 | } |
| 58 | |
| 59 | // Both move doors answer a move of an unplaced --from seed or an |
| 60 | // uncommitted import with the deck's sentence (spec §5.8), before any |
| 61 | // write: no journal, and on the Phase 1 door no claim. |
| 62 | func TestMoveDoorsRefuseAnUnplacedSeed(t *testing.T) { |
| 63 | for _, door := range []struct { |
| 64 | name string |
| 65 | phase1 bool |
| 66 | }{{"legacy", false}, {"phase 1", true}} { |
| 67 | for _, c := range []struct{ kind, state, want string }{ |
| 68 | {seeds.KindReplica, seeds.StateReady, "wake the instance once before moving it"}, |
| 69 | {seeds.KindImport, seeds.StateSwapped, "commit or abort the import first"}, |
| 70 | } { |
| 71 | fx, token := seedMoveFixture(t, "code-seed-move", door.phase1) |
| 72 | putSeededRecord(t, fx.rel, seedMoveHost, "ie", c.kind, c.state) |
| 73 | code, _, raw := rawReq(t, http.MethodPost, instancesURL(fx, release.OpsAccount, "shop")+"/"+seedMoveHost+"/move", token, `{"region":"uk"}`) |
| 74 | if code != http.StatusConflict || errorText(raw) != c.want { |
| 75 | t.Fatalf("%s door, %s %s = %d %s", door.name, c.kind, c.state, code, raw) |
| 76 | } |
| 77 | got, err := provision.GetRecord(context.Background(), fx.rel, release.OpsAccount, "shop", seedMoveHost) |
| 78 | if err != nil || got.Move != nil { |
| 79 | t.Fatalf("%s door: a refused move left a journal: %+v, %v", door.name, got, err) |
| 80 | } |
| 81 | if claim := claimBytes(t, fx, seedMoveHost); claim != "" { |
| 82 | t.Fatalf("%s door: a refused move wrote a claim: %s", door.name, claim) |
| 83 | } |
| 84 | } |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | // A placed seed is not the helper's to refuse: the legacy door goes on to |
| 89 | // its own checks (here, no armed regions). |
| 90 | func TestMoveDoorLetsAPlacedSeedThrough(t *testing.T) { |
| 91 | fx, token := seedMoveFixture(t, "code-seed-move-placed", false) |
| 92 | putSeededRecord(t, fx.rel, seedMoveHost, "ie", seeds.KindReplica, seeds.StatePlaced) |
| 93 | code, _, raw := rawReq(t, http.MethodPost, instancesURL(fx, release.OpsAccount, "shop")+"/"+seedMoveHost+"/move", token, `{"region":"uk"}`) |
| 94 | if code != http.StatusBadRequest || errorText(raw) != "this deployment publishes no armed regions — nothing to move to" { |
| 95 | t.Fatalf("placed seed = %d %s", code, raw) |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | // placeSeedFixture is fleetFixture with the shop app, one route and an |
| 100 | // owned fleet "pi" to place it on. |
| 101 | func placeSeedFixture(t *testing.T, code string) (*consoleFixture, string) { |
| 102 | t.Helper() |
| 103 | fx, token := fleetFixture(t, code) |
| 104 | if _, err := fx.st.CreateAppIn(OpsAccountID, "shop", "paul@keymail.dev", time.Now()); err != nil { |
| 105 | t.Fatal(err) |
| 106 | } |
| 107 | if status, body := postJSON(t, fleetsURL(fx, fx.sqid)+"/new", |
| 108 | map[string]string{"name": "pi", "bucket": "pi-fleet-data"}, token); status != http.StatusOK { |
| 109 | t.Fatalf("create fleet = %d %v", status, body) |
| 110 | } |
| 111 | return fx, token |
| 112 | } |
| 113 | |
| 114 | // assertNotPlaced checks a refused place wrote nothing: the app's placement |
| 115 | // and every record's region are as before. |
| 116 | func assertNotPlaced(t *testing.T, fx *consoleFixture, hosts ...string) { |
| 117 | t.Helper() |
| 118 | if p, err := fx.st.AppPlacement(OpsAccountID, "shop"); err != nil || p != "" { |
| 119 | t.Fatalf("a refused place recorded placement %q, %v", p, err) |
| 120 | } |
| 121 | for _, h := range hosts { |
| 122 | got, err := provision.GetRecord(context.Background(), fx.rel, fx.sqid, "shop", h) |
| 123 | if err != nil || got.Region != "ie" { |
| 124 | t.Fatalf("a refused place re-homed %s: %+v, %v", h, got, err) |
| 125 | } |
| 126 | } |
| 127 | } |
| 128 | |
| 129 | // carlos apps place re-homes every record of the app, so it is a move too |
| 130 | // (preflight M8(a)): an unplaced --from seed or an uncommitted import |
| 131 | // refuses it, with the deck's sentence, before any write. |
| 132 | func TestPlaceRefusesAnUnplacedSeed(t *testing.T) { |
| 133 | for _, c := range []struct{ kind, state, want string }{ |
| 134 | {seeds.KindReplica, seeds.StateReady, "wake the instance once before moving it"}, |
| 135 | {seeds.KindImport, seeds.StatePlaced, "commit or abort the import first"}, |
| 136 | } { |
| 137 | fx, token := placeSeedFixture(t, "code-place-seed") |
| 138 | putSeededRecord(t, fx.rel, "plain.example.com", "ie", seeds.KindReplica, seeds.StatePlaced) |
| 139 | putSeededRecord(t, fx.rel, seedMoveHost, "ie", c.kind, c.state) |
| 140 | code, _, raw := rawReq(t, http.MethodPost, placeURL(fx, "shop"), token, `{"target":"fleet/pi"}`) |
| 141 | if code != http.StatusConflict || errorText(raw) != c.want { |
| 142 | t.Fatalf("%s %s: place = %d %s", c.kind, c.state, code, raw) |
| 143 | } |
| 144 | assertNotPlaced(t, fx, "plain.example.com", seedMoveHost) |
| 145 | } |
| 146 | } |
| 147 | |
| 148 | // A record the place cannot read could be a seed that may not move, so the |
| 149 | // place is refused (it fails closed): a record that will not decode, and a |
| 150 | // read that fails. |
| 151 | func TestPlaceRefusesWhenARecordCannotBeRead(t *testing.T) { |
| 152 | fx, token := placeSeedFixture(t, "code-place-unreadable") |
| 153 | putSeededRecord(t, fx.rel, "plain.example.com", "ie", seeds.KindReplica, seeds.StatePlaced) |
| 154 | bad := provision.RecordKey(fx.sqid, "shop", "broken.example.com") |
| 155 | if _, err := fx.rel.Put(context.Background(), bad, []byte("{not json"), ""); err != nil { |
| 156 | t.Fatal(err) |
| 157 | } |
| 158 | code, _, raw := rawReq(t, http.MethodPost, placeURL(fx, "shop"), token, `{"target":"fleet/pi"}`) |
| 159 | if code != http.StatusBadGateway || errorText(raw) != "could not update this app's instances. Try again." { |
| 160 | t.Fatalf("an undecodable record: place = %d %s", code, raw) |
| 161 | } |
| 162 | assertNotPlaced(t, fx, "plain.example.com") |
| 163 | if err := fx.rel.Delete(context.Background(), bad); err != nil { |
| 164 | t.Fatal(err) |
| 165 | } |
| 166 | |
| 167 | fx.srv.cfg.Releases = &failGetStore{Store: fx.rel, key: provision.RecordKey(fx.sqid, "shop", "plain.example.com")} |
| 168 | code, _, raw = rawReq(t, http.MethodPost, placeURL(fx, "shop"), token, `{"target":"fleet/pi"}`) |
| 169 | if code != http.StatusBadGateway || errorText(raw) != "could not update this app's instances. Try again." { |
| 170 | t.Fatalf("a failed read: place = %d %s", code, raw) |
| 171 | } |
| 172 | assertNotPlaced(t, fx, "plain.example.com") |
| 173 | } |
| 174 | |
| 175 | // A record deleted between the strict read's listing and its read is the |
| 176 | // I4 race one step earlier: the read is taken again, and the place lands. |
| 177 | func TestPlaceReadsAgainWhenARecordVanishes(t *testing.T) { |
| 178 | fx, token := placeSeedFixture(t, "code-place-seed-vanish") |
| 179 | putSeededRecord(t, fx.rel, "plain.example.com", "ie", seeds.KindReplica, seeds.StatePlaced) |
| 180 | putSeededRecord(t, fx.rel, "gone.example.com", "ie", seeds.KindReplica, seeds.StatePlaced) |
| 181 | fx.srv.cfg.Releases = &vanishOnGetStore{Store: fx.rel, vanish: provision.RecordKey(fx.sqid, "shop", "gone.example.com")} |
| 182 | code, _, raw := rawReq(t, http.MethodPost, placeURL(fx, "shop"), token, `{"target":"fleet/pi"}`) |
| 183 | if code != http.StatusOK { |
| 184 | t.Fatalf("place = %d %s", code, raw) |
| 185 | } |
| 186 | got, err := provision.GetRecord(context.Background(), fx.rel, fx.sqid, "shop", "plain.example.com") |
| 187 | if err != nil || got.Region != "fleet/pi" { |
| 188 | t.Fatalf("after the place: %+v, %v", got, err) |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | // The backstop: a seeded record created after the place's read is refused |
| 193 | // under SetRecordRegion's CAS, and the place answers the same sentence. |
| 194 | func TestPlaceAnswersTheBackstopsSentence(t *testing.T) { |
| 195 | fx, token := placeSeedFixture(t, "code-place-seed-backstop") |
| 196 | putSeededRecord(t, fx.rel, "plain.example.com", "ie", seeds.KindReplica, seeds.StatePlaced) |
| 197 | fx.srv.cfg.Releases = &createOnSecondListStore{Store: fx.rel, prefix: provision.RecordsPrefix(fx.sqid, "shop"), create: func(st release.Store) { |
| 198 | putSeededRecord(t, st, seedMoveHost, "ie", seeds.KindImport, seeds.StateSwapped) |
| 199 | }} |
| 200 | code, _, raw := rawReq(t, http.MethodPost, placeURL(fx, "shop"), token, `{"target":"fleet/pi"}`) |
| 201 | if code != http.StatusConflict || errorText(raw) != "commit or abort the import first" { |
| 202 | t.Fatalf("place = %d %s", code, raw) |
| 203 | } |
| 204 | got, err := provision.GetRecord(context.Background(), fx.rel, fx.sqid, "shop", seedMoveHost) |
| 205 | if err != nil || got.Region != "ie" { |
| 206 | t.Fatalf("the late import was re-homed: %+v, %v", got, err) |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | // failGetStore fails every Get of one key. |
| 211 | type failGetStore struct { |
| 212 | release.Store |
| 213 | key string |
| 214 | } |
| 215 | |
| 216 | func (f *failGetStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 217 | if key == f.key { |
| 218 | return nil, "", errors.New("injected read failure") |
| 219 | } |
| 220 | return f.Store.Get(ctx, key) |
| 221 | } |
| 222 | |
| 223 | // createOnSecondListStore runs create once, just before the second List of |
| 224 | // prefix: after the place's strict read, before its walk. |
| 225 | type createOnSecondListStore struct { |
| 226 | release.Store |
| 227 | prefix string |
| 228 | create func(release.Store) |
| 229 | lists int |
| 230 | } |
| 231 | |
| 232 | func (c *createOnSecondListStore) List(ctx context.Context, prefix string) ([]string, error) { |
| 233 | if prefix == c.prefix { |
| 234 | c.lists++ |
| 235 | if c.lists == 2 { |
| 236 | c.create(c.Store) |
| 237 | } |
| 238 | } |
| 239 | return c.Store.List(ctx, prefix) |
| 240 | } |
| 241 | |
amadan