| 1 | package provision |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "reflect" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/registry" |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | "github.com/carlosframework/platform/internal/seeds" |
| 14 | ) |
| 15 | |
| 16 | // shopSetup is the import tests' neutral fixture: an account manifest |
| 17 | // (id acctID) and its app "shop" (domain shop.example.com, the shared |
| 18 | // pool, no seed command). |
| 19 | func shopSetup(t *testing.T) (*release.DirStore, string) { |
| 20 | t.Helper() |
| 21 | s := dirStore(t) |
| 22 | sqid := writeAccountManifest(t, s, acctID, "Shop") |
| 23 | if err := PutManifest(ctx, s, sqid, "shop", Manifest{Schema: 1, App: "shop", Domain: "shop.example.com"}); err != nil { |
| 24 | t.Fatal(err) |
| 25 | } |
| 26 | return s, sqid |
| 27 | } |
| 28 | |
| 29 | func shopRecord(sqid string) Record { |
| 30 | return Record{Schema: 1, Host: "a.shop.example.com", Slug: "a", App: "shop", |
| 31 | Account: sqid, Region: "ie", Channel: "stable", CreatedAt: time.Now()} |
| 32 | } |
| 33 | |
| 34 | func newImport(sqid string) Record { |
| 35 | rec := shopRecord(sqid) |
| 36 | id, _ := seeds.NewID() |
| 37 | rec.Seed = &SeedSource{ID: id, Kind: seeds.KindImport, State: seeds.StateReady, SHA256: strings.Repeat("a", 64), Bytes: 10, |
| 38 | Replaces: "127.0.0.1:8401", VerifyHash: strings.Repeat("b", 64), By: "t", At: time.Now()} |
| 39 | return rec |
| 40 | } |
| 41 | |
| 42 | func importRecord(t *testing.T, s release.Store, sqid string) Record { |
| 43 | t.Helper() |
| 44 | rec := newImport(sqid) |
| 45 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 46 | t.Fatal(err) |
| 47 | } |
| 48 | return rec |
| 49 | } |
| 50 | |
| 51 | // addrRow is the legacy row an import replaces: an addr route to the |
| 52 | // app's own process, not minted from any record. |
| 53 | func addrRow(rec Record, account string) registry.Route { |
| 54 | return registry.Route{Host: rec.Host, Kind: "service", Addr: "127.0.0.1:8401", App: rec.App, Account: account, |
| 55 | Region: "ie", Channel: "stable", Status: "ready", Unit: "shop@a.service"} |
| 56 | } |
| 57 | |
| 58 | func importCfg(t *testing.T, owners map[string]seeds.LegacyOwner) ReconcileConfig { |
| 59 | t.Helper() |
| 60 | cfg := nestedSeedCfg(dirStore(t)) |
| 61 | cfg.Pools = nil |
| 62 | cfg.Seeds.LegacyOwners = func(context.Context) (*seeds.LegacyOwners, error) { |
| 63 | return &seeds.LegacyOwners{V: 1, Names: owners}, nil |
| 64 | } |
| 65 | // The host has no env file before the import, as a host the platform |
| 66 | // never ran has none. |
| 67 | cfg.Seeds.EnvBefore = func(string, string) (string, error) { return registry.EnvBeforeAbsent, nil } |
| 68 | return cfg |
| 69 | } |
| 70 | |
| 71 | func seedState(t *testing.T, s release.Store, rec Record) *SeedSource { |
| 72 | t.Helper() |
| 73 | r, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host) |
| 74 | if err != nil { |
| 75 | t.Fatal(err) |
| 76 | } |
| 77 | return r.Seed |
| 78 | } |
| 79 | |
| 80 | // wantUntouched asserts a deferred swap: the record still ready with no |
| 81 | // reason, the stored row exactly row, and no journal. |
| 82 | func wantUntouched(t *testing.T, s release.Store, reg *registry.Registry, rec Record, row *registry.Route) { |
| 83 | t.Helper() |
| 84 | if got := seedState(t, s, rec); got.State != seeds.StateReady || got.Reason != "" { |
| 85 | t.Fatalf("seed = %s %q, want ready (deferred, not refused)", got.State, got.Reason) |
| 86 | } |
| 87 | got, err := reg.Get(rec.Host) |
| 88 | if err != nil { |
| 89 | t.Fatal(err) |
| 90 | } |
| 91 | if row == nil { |
| 92 | if got != nil { |
| 93 | t.Fatalf("a row appeared: %+v", got) |
| 94 | } |
| 95 | } else if got == nil || !reflect.DeepEqual(*got, *row) { |
| 96 | t.Fatalf("row = %+v, want it unchanged %+v", got, row) |
| 97 | } |
| 98 | if j, _ := reg.ImportJournal(rec.Host); j != nil { |
| 99 | t.Fatalf("a deferred import left a journal row: %+v", j) |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | func TestSwapReplacesTheAddrRowAndJournalsIt(t *testing.T) { |
| 104 | s, sqid := shopSetup(t) |
| 105 | rec := importRecord(t, s, sqid) |
| 106 | reg := regWith(t, addrRow(rec, sqid)) |
| 107 | original := mustGet(t, reg, rec.Host) |
| 108 | cfg := importCfg(t, nil) |
| 109 | at := time.Unix(1_800_000_000, 0) |
| 110 | cfg.Now = func() time.Time { return at } |
| 111 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 112 | t.Fatal(err) |
| 113 | } |
| 114 | got := mustGet(t, reg, rec.Host) |
| 115 | if got.Kind != "instance" || got.Socket != NestedSocket(rec.Host) || got.Addr != "" || got.Addrs != "" || |
| 116 | got.ImportGate != rec.Seed.VerifyHash || got.DBPath != NestedDBPath("/data", rec.Host) || got.Account != sqid || |
| 117 | !got.Provisioned || got.Seed != "" || !got.Hibernate { |
| 118 | t.Fatalf("swapped row = %+v", got) |
| 119 | } |
| 120 | if col, _ := seeds.ParseRowSeed(got.SeedSource); col == nil || col.ID != rec.Seed.ID || col.State != seeds.StateSwapped { |
| 121 | t.Fatalf("swapped row's seed column = %+v, want this seed at swapped", col) |
| 122 | } |
| 123 | j, err := reg.ImportJournal(rec.Host) |
| 124 | if err != nil || j == nil || j.SeedID != rec.Seed.ID || j.State != registry.JournalSwapped || !reflect.DeepEqual(j.Original, *original) || |
| 125 | j.RecordKey != RecordKey(rec.Account, rec.App, rec.Host) || j.At != at.Unix() { |
| 126 | t.Fatalf("journal = %+v, %v", j, err) |
| 127 | } |
| 128 | if st := seedState(t, s, rec); st.State != seeds.StateSwapped { |
| 129 | t.Fatalf("record = %s", st.State) |
| 130 | } |
| 131 | |
| 132 | // The next pass converges the row as an ordinary owned row and leaves |
| 133 | // the gate and the journal alone. |
| 134 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 135 | t.Fatal(err) |
| 136 | } |
| 137 | again := mustGet(t, reg, rec.Host) |
| 138 | if again.ImportGate != rec.Seed.VerifyHash || again.Socket != got.Socket || again.DBPath != got.DBPath || again.Addr != "" { |
| 139 | t.Fatalf("row after the next pass = %+v", again) |
| 140 | } |
| 141 | if j2, _ := reg.ImportJournal(rec.Host); j2 == nil || !reflect.DeepEqual(*j2, *j) { |
| 142 | t.Fatalf("journal after the next pass = %+v", j2) |
| 143 | } |
| 144 | } |
| 145 | |
| 146 | // A row backed by an addrs set of exactly Replaces is the same address. |
| 147 | func TestSwapAcceptsAnAddrsRowOfExactlyReplaces(t *testing.T) { |
| 148 | s, sqid := shopSetup(t) |
| 149 | rec := importRecord(t, s, sqid) |
| 150 | row := addrRow(rec, sqid) |
| 151 | row.Addr, row.Addrs = "", "127.0.0.1:8401" |
| 152 | reg := regWith(t, row) |
| 153 | if err := Reconcile(ctx, s, reg, importCfg(t, nil)); err != nil { |
| 154 | t.Fatal(err) |
| 155 | } |
| 156 | if got := mustGet(t, reg, rec.Host); got.ImportGate != rec.Seed.VerifyHash || got.Kind != "instance" { |
| 157 | t.Fatalf("row = %+v", got) |
| 158 | } |
| 159 | } |
| 160 | |
| 161 | func TestSwapRefusesEachCondition(t *testing.T) { |
| 162 | for _, c := range []struct { |
| 163 | name string |
| 164 | row func(Record, string) *registry.Route |
| 165 | owners map[string]seeds.LegacyOwner |
| 166 | cfg func(*ReconcileConfig) |
| 167 | want string |
| 168 | }{ |
| 169 | {"no row", func(Record, string) *registry.Route { return nil }, nil, nil, "there is no route at a.shop.example.com to replace"}, |
| 170 | {"addr changed", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Addr = "127.0.0.1:9"; return &x }, nil, nil, |
| 171 | "the route's address is 127.0.0.1:9, not 127.0.0.1:8401"}, |
| 172 | {"addrs wider than replaces", func(r Record, a string) *registry.Route { |
| 173 | x := addrRow(r, a) |
| 174 | x.Addr, x.Addrs = "", "127.0.0.1:8401,127.0.0.1:8402" |
| 175 | return &x |
| 176 | }, nil, nil, "the route's address is 127.0.0.1:8401,127.0.0.1:8402, not 127.0.0.1:8401"}, |
| 177 | {"addrs behind a matching addr", func(r Record, a string) *registry.Route { |
| 178 | x := addrRow(r, a) |
| 179 | x.Addrs = "127.0.0.1:9" |
| 180 | return &x |
| 181 | }, nil, nil, "the route's address is 127.0.0.1:9, not 127.0.0.1:8401"}, |
| 182 | {"another app", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.App = "other"; return &x }, nil, nil, |
| 183 | "the route belongs to app other"}, |
| 184 | {"a third account", func(r Record, _ string) *registry.Route { x := addrRow(r, "zzz"); return &x }, nil, nil, |
| 185 | "the route is filed under another account"}, |
| 186 | {"operator row, no assignment", func(r Record, _ string) *registry.Route { x := addrRow(r, release.OpsAccount); return &x }, nil, nil, |
| 187 | "the route is filed under the operator account, and no assignment maps shop to this app"}, |
| 188 | {"legacy operator row, no assignment", func(r Record, _ string) *registry.Route { x := addrRow(r, ""); return &x }, nil, nil, |
| 189 | "no assignment maps shop to this app"}, |
| 190 | {"operator row, assigned to another account", func(r Record, _ string) *registry.Route { x := addrRow(r, release.OpsAccount); return &x }, |
| 191 | map[string]seeds.LegacyOwner{"shop": {Account: "other", App: "shop"}}, nil, "no assignment maps shop to this app"}, |
| 192 | {"operator row, assigned to another app", func(r Record, _ string) *registry.Route { x := addrRow(r, release.OpsAccount); return &x }, |
| 193 | map[string]seeds.LegacyOwner{"shop": {Account: "SELF", App: "store"}}, nil, "no assignment maps shop to this app"}, |
| 194 | {"socket backed", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Socket = "/run/x.sock"; return &x }, nil, nil, "(socket)"}, |
| 195 | {"no address", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Addr = ""; return &x }, nil, nil, "(no address)"}, |
| 196 | {"provisioned", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Provisioned = true; return &x }, nil, nil, "(provisioned)"}, |
| 197 | {"static", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Kind = "static"; return &x }, nil, nil, "(kind static)"}, |
| 198 | {"backing", func(r Record, a string) *registry.Route { |
| 199 | x := addrRow(r, a) |
| 200 | x.Backing = registry.BackingUnit |
| 201 | return &x |
| 202 | }, nil, nil, "(backing)"}, |
| 203 | {"tunnel", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Tunnel = "t"; return &x }, nil, nil, "(tunnel)"}, |
| 204 | {"caps", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Caps = "host-role"; return &x }, nil, nil, "(caps)"}, |
| 205 | {"platform", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Platform = true; return &x }, nil, nil, "(platform)"}, |
| 206 | {"canary", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Canary = true; return &x }, nil, nil, "(canary)"}, |
| 207 | {"sidecar", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Kind = "sidecar"; return &x }, nil, nil, "(kind sidecar)"}, |
| 208 | // The edge serves these from somewhere other than the addr: a |
| 209 | // fleet box over the reverse channel, or the activator's own child. |
| 210 | {"fleet placed", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Region = "fleet/edge1"; return &x }, nil, nil, "(region)"}, |
| 211 | {"fleet box", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.FleetBox = "box1"; return &x }, nil, nil, "(fleet_box)"}, |
| 212 | {"hibernating instance", func(r Record, a string) *registry.Route { |
| 213 | x := addrRow(r, a) |
| 214 | x.Kind, x.Hibernate = "instance", true |
| 215 | return &x |
| 216 | }, nil, nil, "(hibernate)"}, |
| 217 | {"claim owned", func(r Record, a string) *registry.Route { |
| 218 | x := addrRow(r, a) |
| 219 | x.ClaimOwnership = registry.ClaimOwnership{ClaimIncarnation: "inc", ClaimStamp: 1, ClaimOwner: "owner", ClaimRowID: "row"} |
| 220 | return &x |
| 221 | }, nil, nil, "(claim_owner)"}, |
| 222 | {"flat", func(r Record, a string) *registry.Route { x := addrRow(r, a); return &x }, nil, func(c *ReconcileConfig) { c.Layout = nil }, |
| 223 | "this box does not run instances nested"}, |
| 224 | {"flat intent", func(r Record, a string) *registry.Route { x := addrRow(r, a); return &x }, nil, func(c *ReconcileConfig) { |
| 225 | c.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil } |
| 226 | }, "this box does not run instances nested"}, |
| 227 | {"claimed", func(r Record, a string) *registry.Route { x := addrRow(r, a); return &x }, nil, func(c *ReconcileConfig) { c.ClaimEnabled = true }, |
| 228 | "claim control is on for this box"}, |
| 229 | } { |
| 230 | t.Run(c.name, func(t *testing.T) { |
| 231 | s, sqid := shopSetup(t) |
| 232 | rec := importRecord(t, s, sqid) |
| 233 | var reg *registry.Registry |
| 234 | var before *registry.Route |
| 235 | if row := c.row(rec, sqid); row != nil { |
| 236 | reg = regWith(t, *row) |
| 237 | before = mustGet(t, reg, rec.Host) |
| 238 | } else { |
| 239 | reg = regWith(t) |
| 240 | } |
| 241 | owners := c.owners |
| 242 | for k, v := range owners { // "SELF" is the record's own account |
| 243 | if v.Account == "SELF" { |
| 244 | owners[k] = seeds.LegacyOwner{Account: sqid, App: v.App} |
| 245 | } |
| 246 | } |
| 247 | cfg := importCfg(t, owners) |
| 248 | if c.cfg != nil { |
| 249 | c.cfg(&cfg) |
| 250 | } |
| 251 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 252 | t.Fatal(err) |
| 253 | } |
| 254 | if st := seedState(t, s, rec); st.State != seeds.StateRefused || !strings.Contains(st.Reason, c.want) { |
| 255 | t.Fatalf("seed = %s %q, want refused with %q", st.State, st.Reason, c.want) |
| 256 | } |
| 257 | if j, _ := reg.ImportJournal(rec.Host); j != nil { |
| 258 | t.Fatalf("a refused import left a journal row: %+v", j) |
| 259 | } |
| 260 | after, _ := reg.Get(rec.Host) |
| 261 | if (before == nil) != (after == nil) || (before != nil && !reflect.DeepEqual(*before, *after)) { |
| 262 | t.Fatalf("a refused import changed the row: %+v -> %+v", before, after) |
| 263 | } |
| 264 | // A refused import is skipped from then on. |
| 265 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 266 | t.Fatal(err) |
| 267 | } |
| 268 | if st := seedState(t, s, rec); st.State != seeds.StateRefused { |
| 269 | t.Fatalf("seed after the next pass = %s", st.State) |
| 270 | } |
| 271 | }) |
| 272 | } |
| 273 | } |
| 274 | |
| 275 | // M5: an operator-filed row ("bab", or a legacy row with no account) is |
| 276 | // accepted only through the assignment, and the new row is filed under |
| 277 | // the record's account. |
| 278 | func TestSwapAcceptsAnAssignedOperatorRow(t *testing.T) { |
| 279 | for _, acct := range []string{release.OpsAccount, ""} { |
| 280 | t.Run("account "+acct, func(t *testing.T) { |
| 281 | s, sqid := shopSetup(t) |
| 282 | rec := importRecord(t, s, sqid) |
| 283 | reg := regWith(t, addrRow(rec, acct)) |
| 284 | if err := Reconcile(ctx, s, reg, importCfg(t, map[string]seeds.LegacyOwner{"shop": {Account: sqid, App: "shop"}})); err != nil { |
| 285 | t.Fatal(err) |
| 286 | } |
| 287 | if got := mustGet(t, reg, rec.Host); got.Account != sqid || got.Kind != "instance" || got.ImportGate == "" { |
| 288 | t.Fatalf("row = %+v; want it filed under the record's account, gated", got) |
| 289 | } |
| 290 | if st := seedState(t, s, rec); st.State != seeds.StateSwapped { |
| 291 | t.Fatalf("record = %s", st.State) |
| 292 | } |
| 293 | }) |
| 294 | } |
| 295 | } |
| 296 | |
| 297 | // The assignment is an authority check: an unreadable file or an unwired |
| 298 | // reader defers the swap, never refuses it and never accepts the row. |
| 299 | func TestSwapDefersWhenTheAssignmentCannotBeRead(t *testing.T) { |
| 300 | for name, hook := range map[string]func(context.Context) (*seeds.LegacyOwners, error){ |
| 301 | "read error": func(context.Context) (*seeds.LegacyOwners, error) { return nil, errors.New("store down") }, |
| 302 | "unwired": nil, |
| 303 | } { |
| 304 | t.Run(name, func(t *testing.T) { |
| 305 | s, sqid := shopSetup(t) |
| 306 | rec := importRecord(t, s, sqid) |
| 307 | reg := regWith(t, addrRow(rec, release.OpsAccount)) |
| 308 | row := mustGet(t, reg, rec.Host) |
| 309 | cfg := importCfg(t, nil) |
| 310 | cfg.Seeds.LegacyOwners = hook |
| 311 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 312 | t.Fatal(err) |
| 313 | } |
| 314 | wantUntouched(t, s, reg, rec, row) |
| 315 | }) |
| 316 | } |
| 317 | } |
| 318 | |
| 319 | // M3: the swap is skipped, the record left as it is, unless the verify |
| 320 | // hash is a hash, the host and seed id pass seeds.Check, and the new row |
| 321 | // is nested and hibernating. |
| 322 | func TestSwapSkipsARecordItCannotGate(t *testing.T) { |
| 323 | for _, c := range []struct { |
| 324 | name string |
| 325 | edit func(*Record) |
| 326 | }{ |
| 327 | {"empty verify hash", func(r *Record) { r.Seed.VerifyHash = "" }}, |
| 328 | {"verify hash not a hash", func(r *Record) { r.Seed.VerifyHash = "closed" }}, |
| 329 | {"verify hash in capitals", func(r *Record) { r.Seed.VerifyHash = strings.Repeat("B", 64) }}, |
| 330 | {"bad seed id", func(r *Record) { r.Seed.ID = "sd_../../x" }}, |
| 331 | {"bad host", func(r *Record) { r.Host = "A.shop.example.com/.." }}, |
| 332 | {"does not hibernate", func(r *Record) { off := false; r.Hibernate = &off }}, |
| 333 | {"not an instance", func(r *Record) { r.Kind = "service" }}, |
| 334 | {"proxies to an address", func(r *Record) { r.Addr = "127.0.0.1:9000" }}, |
| 335 | } { |
| 336 | t.Run(c.name, func(t *testing.T) { |
| 337 | s, sqid := shopSetup(t) |
| 338 | stored := importRecord(t, s, sqid) |
| 339 | rec := stored |
| 340 | seed := *stored.Seed |
| 341 | rec.Seed = &seed |
| 342 | c.edit(&rec) |
| 343 | reg := regWith(t, addrRow(stored, sqid)) |
| 344 | row := mustGet(t, reg, stored.Host) |
| 345 | cfg := importCfg(t, nil) |
| 346 | got, _ := reg.Get(stored.Host) |
| 347 | swapImport(ctx, s, reg, cfg, rec, got, mintLayoutReader(ctx, s, cfg), nil) |
| 348 | wantUntouched(t, s, reg, stored, row) |
| 349 | }) |
| 350 | } |
| 351 | } |
| 352 | |
| 353 | // Each skip is importUngateable's own (not a later guard's), and a sound |
| 354 | // import record passes it. |
| 355 | func TestImportUngateable(t *testing.T) { |
| 356 | good := newImport("abc") |
| 357 | if why := importUngateable(good, importRow(good, "/data", nil), "/data"); why != "" { |
| 358 | t.Fatalf("a sound record: %q", why) |
| 359 | } |
| 360 | for name, edit := range map[string]func(*Record){ |
| 361 | "empty verify hash": func(r *Record) { r.Seed.VerifyHash = "" }, |
| 362 | "closed gate": func(r *Record) { r.Seed.VerifyHash = "closed" }, |
| 363 | "bad seed id": func(r *Record) { r.Seed.ID = "sd_../../x" }, |
| 364 | "bad host": func(r *Record) { r.Host = "A.shop.example.com/.." }, |
| 365 | "does not hibernate": func(r *Record) { |
| 366 | off := false |
| 367 | r.Hibernate = &off |
| 368 | }, |
| 369 | "not an instance": func(r *Record) { r.Kind = "service" }, |
| 370 | } { |
| 371 | rec := newImport("abc") |
| 372 | seed := *rec.Seed |
| 373 | rec.Seed = &seed |
| 374 | edit(&rec) |
| 375 | if why := importUngateable(rec, importRow(rec, "/data", nil), "/data"); why == "" { |
| 376 | t.Errorf("%s: not skipped", name) |
| 377 | } |
| 378 | } |
| 379 | } |
| 380 | |
| 381 | // The same rule through Reconcile, for a record shape CreateRecord stores. |
| 382 | func TestReconcileDoesNotSwapAnImportThatWouldNotHibernate(t *testing.T) { |
| 383 | s, sqid := shopSetup(t) |
| 384 | rec := newImport(sqid) |
| 385 | off := false |
| 386 | rec.Hibernate = &off |
| 387 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 388 | t.Fatal(err) |
| 389 | } |
| 390 | reg := regWith(t, addrRow(rec, sqid)) |
| 391 | row := mustGet(t, reg, rec.Host) |
| 392 | if err := Reconcile(ctx, s, reg, importCfg(t, nil)); err != nil { |
| 393 | t.Fatal(err) |
| 394 | } |
| 395 | wantUntouched(t, s, reg, rec, row) |
| 396 | } |
| 397 | |
| 398 | // M4: a layout read error or a NestedOK error defers to the next pass and |
| 399 | // writes nothing; only a layout intent that says flat refuses (above). |
| 400 | func TestSwapDefersATransientLayoutAnswer(t *testing.T) { |
| 401 | for _, c := range []struct { |
| 402 | name string |
| 403 | cfg func(*ReconcileConfig) |
| 404 | }{ |
| 405 | {"layout read error", func(c *ReconcileConfig) { |
| 406 | c.Layout = func(context.Context, release.Store) (string, error) { return "", errors.New("store down") } |
| 407 | }}, |
| 408 | {"nested probe error", func(c *ReconcileConfig) { |
| 409 | c.Seeds.NestedOK = func() error { return errors.New("ready probe timed out") } |
| 410 | }}, |
| 411 | } { |
| 412 | t.Run(c.name, func(t *testing.T) { |
| 413 | s, sqid := shopSetup(t) |
| 414 | rec := importRecord(t, s, sqid) |
| 415 | reg := regWith(t, addrRow(rec, sqid)) |
| 416 | row := mustGet(t, reg, rec.Host) |
| 417 | cfg := importCfg(t, nil) |
| 418 | c.cfg(&cfg) |
| 419 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 420 | t.Fatal(err) |
| 421 | } |
| 422 | wantUntouched(t, s, reg, rec, row) |
| 423 | |
| 424 | // The next pass, with the box answering, swaps. |
| 425 | if err := Reconcile(ctx, s, reg, importCfg(t, nil)); err != nil { |
| 426 | t.Fatal(err) |
| 427 | } |
| 428 | if st := seedState(t, s, rec); st.State != seeds.StateSwapped { |
| 429 | t.Fatalf("record after the box answered = %s %q", st.State, st.Reason) |
| 430 | } |
| 431 | }) |
| 432 | } |
| 433 | } |
| 434 | |
| 435 | func TestSwapRunsWhenTheNestedProbeAnswers(t *testing.T) { |
| 436 | s, sqid := shopSetup(t) |
| 437 | rec := importRecord(t, s, sqid) |
| 438 | reg := regWith(t, addrRow(rec, sqid)) |
| 439 | cfg := importCfg(t, nil) |
| 440 | asked := 0 |
| 441 | cfg.Seeds.NestedOK = func() error { asked++; return nil } |
| 442 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 443 | t.Fatal(err) |
| 444 | } |
| 445 | if asked != 1 || mustGet(t, reg, rec.Host).ImportGate == "" { |
| 446 | t.Fatalf("probe asked %d times, row %+v", asked, mustGet(t, reg, rec.Host)) |
| 447 | } |
| 448 | } |
| 449 | |
| 450 | // Transient refusals of the write itself defer: the config gate, a row |
| 451 | // held by a running operation, and a dry run. |
| 452 | func TestSwapDefersWhatTheWriteCannotTakeYet(t *testing.T) { |
| 453 | for _, c := range []struct { |
| 454 | name string |
| 455 | held bool |
| 456 | cfg func(*ReconcileConfig) |
| 457 | }{ |
| 458 | {"config gate", false, func(c *ReconcileConfig) { |
| 459 | c.ConfigGate = func(context.Context, registry.Route) error { return errors.New("no key yet") } |
| 460 | }}, |
| 461 | {"held", true, nil}, |
| 462 | {"dry run", false, func(c *ReconcileConfig) { c.DryRun = true }}, |
| 463 | } { |
| 464 | t.Run(c.name, func(t *testing.T) { |
| 465 | s, sqid := shopSetup(t) |
| 466 | rec := importRecord(t, s, sqid) |
| 467 | r := addrRow(rec, sqid) |
| 468 | if c.held { |
| 469 | // A wrong address too: without the hold's deferral this row |
| 470 | // would be refused SD-N92, not left for the next pass. |
| 471 | r.Addr = "127.0.0.1:9" |
| 472 | } |
| 473 | reg := regWith(t, r) |
| 474 | if c.held { |
| 475 | if err := reg.SetHold(rec.Host, "moving", "op_1"); err != nil { |
| 476 | t.Fatal(err) |
| 477 | } |
| 478 | } |
| 479 | row := mustGet(t, reg, rec.Host) |
| 480 | if c.held && row.HoldOp == "" { |
| 481 | t.Fatalf("fixture: the row is not held: %+v", row) |
| 482 | } |
| 483 | cfg := importCfg(t, nil) |
| 484 | if c.cfg != nil { |
| 485 | c.cfg(&cfg) |
| 486 | } |
| 487 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 488 | t.Fatal(err) |
| 489 | } |
| 490 | wantUntouched(t, s, reg, rec, row) |
| 491 | }) |
| 492 | } |
| 493 | } |
| 494 | |
| 495 | // A crash between the swap's transaction and the record's CAS: the |
| 496 | // journal names this seed and the record still says ready. The next pass |
| 497 | // sees the journal first, never judges the row it made as a legacy row, |
| 498 | // and only redoes the CAS. |
| 499 | func TestSwapResumesTheRecordCASAfterACrash(t *testing.T) { |
| 500 | s, sqid := shopSetup(t) |
| 501 | rec := importRecord(t, s, sqid) |
| 502 | reg := regWith(t, addrRow(rec, sqid)) |
| 503 | original := mustGet(t, reg, rec.Host) |
| 504 | next := RouteFor(rec, "/data", "", true, nil) |
| 505 | next.ImportGate = rec.Seed.VerifyHash |
| 506 | if err := reg.SwapForImport(registry.ImportJournal{Host: rec.Host, SeedID: rec.Seed.ID, Original: *original, |
| 507 | RecordKey: RecordKey(rec.Account, rec.App, rec.Host), At: 1}, next); err != nil { |
| 508 | t.Fatal(err) |
| 509 | } |
| 510 | swapped := mustGet(t, reg, rec.Host) |
| 511 | j, _ := reg.ImportJournal(rec.Host) |
| 512 | |
| 513 | if err := Reconcile(ctx, s, reg, importCfg(t, nil)); err != nil { |
| 514 | t.Fatal(err) |
| 515 | } |
| 516 | if st := seedState(t, s, rec); st.State != seeds.StateSwapped || st.Reason != "" { |
| 517 | t.Fatalf("record = %s %q, want swapped", st.State, st.Reason) |
| 518 | } |
| 519 | if got := mustGet(t, reg, rec.Host); !reflect.DeepEqual(*got, *swapped) { |
| 520 | t.Fatalf("row changed on the resume: %+v -> %+v", swapped, got) |
| 521 | } |
| 522 | if j2, _ := reg.ImportJournal(rec.Host); j2 == nil || !reflect.DeepEqual(*j2, *j) { |
| 523 | t.Fatalf("journal changed on the resume: %+v -> %+v", j, j2) |
| 524 | } |
| 525 | } |
| 526 | |
| 527 | // A journal row naming this seed ID under another record key is not this |
| 528 | // record's swap: no CAS for this record. The journal's own record (under |
| 529 | // the other key) is gone, so the journal rolls its import back: the |
| 530 | // original route returns exactly, and this record, which holds the host, |
| 531 | // keeps the host's replica and its own state. |
| 532 | func TestSwapResumeNeedsTheRecordKey(t *testing.T) { |
| 533 | s, sqid := shopSetup(t) |
| 534 | rec := importRecord(t, s, sqid) |
| 535 | reg := regWith(t, addrRow(rec, sqid)) |
| 536 | original := mustGet(t, reg, rec.Host) |
| 537 | next := importRow(rec, "/data", nil) |
| 538 | if err := reg.SwapForImport(registry.ImportJournal{Host: rec.Host, SeedID: rec.Seed.ID, Original: *original, |
| 539 | RecordKey: RecordKey(sqid, "other", rec.Host), At: 1}, next); err != nil { |
| 540 | t.Fatal(err) |
| 541 | } |
| 542 | cfg := importCfg(t, nil) |
| 543 | b := newRollbackBox(t, reg, *original) |
| 544 | b.hooks(cfg.Seeds) |
| 545 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 546 | t.Fatal(err) |
| 547 | } |
| 548 | if st := seedState(t, s, rec); st.State != seeds.StateReady || st.Reason != "" { |
| 549 | t.Fatalf("record = %s %q, want ready", st.State, st.Reason) |
| 550 | } |
| 551 | if got := mustGet(t, reg, rec.Host); !reflect.DeepEqual(*got, *original) { |
| 552 | t.Fatalf("row = %+v, want the original", got) |
| 553 | } |
| 554 | if j, _ := reg.ImportJournal(rec.Host); j != nil { |
| 555 | t.Fatalf("journal = %+v", j) |
| 556 | } |
| 557 | if b.local || !b.replica { |
| 558 | t.Fatalf("local %v replica %v: steps %q", b.local, b.replica, b.steps) |
| 559 | } |
| 560 | } |
| 561 | |
| 562 | // A journal row this record's seed does not own (another seed's, or one |
| 563 | // already rolling back) leaves the record alone: no refusal, no swap. This |
| 564 | // is the swap's own backstop (seedswap.go), called directly: in a pass the |
| 565 | // journal pass decides such a row first, and rolls it back. |
| 566 | func TestSwapLeavesAnotherSeedsJournalAlone(t *testing.T) { |
| 567 | s, sqid := shopSetup(t) |
| 568 | rec := importRecord(t, s, sqid) |
| 569 | reg := regWith(t, addrRow(rec, sqid)) |
| 570 | original := mustGet(t, reg, rec.Host) |
| 571 | other := rec |
| 572 | seed := *rec.Seed |
| 573 | seed.ID, _ = seeds.NewID() |
| 574 | other.Seed = &seed |
| 575 | if err := reg.SwapForImport(registry.ImportJournal{Host: rec.Host, SeedID: seed.ID, Original: *original, |
| 576 | RecordKey: RecordKey(rec.Account, rec.App, rec.Host), At: 1}, importRow(other, "/data", nil)); err != nil { |
| 577 | t.Fatal(err) |
| 578 | } |
| 579 | row := mustGet(t, reg, rec.Host) |
| 580 | j, _ := reg.ImportJournal(rec.Host) |
| 581 | swapImport(ctx, s, reg, importCfg(t, nil), rec, row, func() (bool, error) { return true, nil }, nil) |
| 582 | if st := seedState(t, s, rec); st.State != seeds.StateReady || st.Reason != "" { |
| 583 | t.Fatalf("record = %s %q, want ready", st.State, st.Reason) |
| 584 | } |
| 585 | if got := mustGet(t, reg, rec.Host); !reflect.DeepEqual(*got, *row) { |
| 586 | t.Fatalf("row changed: %+v", got) |
| 587 | } |
| 588 | if j2, _ := reg.ImportJournal(rec.Host); !reflect.DeepEqual(j2, j) { |
| 589 | t.Fatalf("journal changed: %+v", j2) |
| 590 | } |
| 591 | } |
| 592 | |
| 593 | // M2: before committed, an import's row is made only by the swap. A box |
| 594 | // with no row never mints one for a swapped or placed import, and neither |
| 595 | // does the swapping box after its gated row was removed. |
| 596 | func TestReconcileNeverMintsAnUncommittedImport(t *testing.T) { |
| 597 | for _, state := range []string{seeds.StateSwapped, seeds.StatePlaced} { |
| 598 | t.Run("no row, "+state, func(t *testing.T) { |
| 599 | s, sqid := shopSetup(t) |
| 600 | rec := newImport(sqid) |
| 601 | rec.Seed.State = state |
| 602 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 603 | t.Fatal(err) |
| 604 | } |
| 605 | reg := regWith(t) |
| 606 | if err := Reconcile(ctx, s, reg, importCfg(t, nil)); err != nil { |
| 607 | t.Fatal(err) |
| 608 | } |
| 609 | if rt, _ := reg.Get(rec.Host); rt != nil { |
| 610 | t.Fatalf("minted %+v", rt) |
| 611 | } |
| 612 | }) |
| 613 | } |
| 614 | t.Run("after the gated row was removed", func(t *testing.T) { |
| 615 | s, sqid := shopSetup(t) |
| 616 | rec := importRecord(t, s, sqid) |
| 617 | reg := regWith(t, addrRow(rec, sqid)) |
| 618 | cfg := importCfg(t, nil) |
| 619 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 620 | t.Fatal(err) |
| 621 | } |
| 622 | if st := seedState(t, s, rec); st.State != seeds.StateSwapped { |
| 623 | t.Fatalf("record = %s", st.State) |
| 624 | } |
| 625 | if err := reg.Delete(rec.Host); err != nil { // carlos remove |
| 626 | t.Fatal(err) |
| 627 | } |
| 628 | for _, mut := range []func(*Record) error{nil, SeedPlace(rec.Seed.ID)} { |
| 629 | if mut != nil { |
| 630 | if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, mut); err != nil { |
| 631 | t.Fatal(err) |
| 632 | } |
| 633 | } |
| 634 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 635 | t.Fatal(err) |
| 636 | } |
| 637 | if rt, _ := reg.Get(rec.Host); rt != nil { |
| 638 | t.Fatalf("%s: minted %+v", seedState(t, s, rec).State, rt) |
| 639 | } |
| 640 | } |
| 641 | }) |
| 642 | } |
| 643 | |
| 644 | // After commit the import is an ordinary instance: a box with no row |
| 645 | // mints it (ungated), as any other record. |
| 646 | func TestReconcileMintsACommittedImport(t *testing.T) { |
| 647 | s, sqid := shopSetup(t) |
| 648 | rec := newImport(sqid) |
| 649 | rec.Seed.State = seeds.StateCommitted |
| 650 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 651 | t.Fatal(err) |
| 652 | } |
| 653 | reg := regWith(t) |
| 654 | if err := Reconcile(ctx, s, reg, importCfg(t, nil)); err != nil { |
| 655 | t.Fatal(err) |
| 656 | } |
| 657 | if got := mustGet(t, reg, rec.Host); got.ImportGate != "" || got.DBPath != NestedDBPath("/data", rec.Host) { |
| 658 | t.Fatalf("row = %+v", got) |
| 659 | } |
| 660 | } |
| 661 | |
| 662 | func TestImportRefusalOrder(t *testing.T) { |
| 663 | rec := newImport("abc") |
| 664 | row := addrRow(rec, "zzz") |
| 665 | row.App = "other" |
| 666 | cfg := ReconcileConfig{ClaimEnabled: true} |
| 667 | if got := importRefusal(rec, &row, cfg, nil, false); got != "claim control is on for this box" { |
| 668 | t.Fatalf("claims first: %q", got) |
| 669 | } |
| 670 | if got := importRefusal(rec, &row, ReconcileConfig{}, nil, false); got != "this box does not run instances nested" { |
| 671 | t.Fatalf("then layout: %q", got) |
| 672 | } |
| 673 | if got := importRefusal(rec, nil, ReconcileConfig{}, nil, true); got != "there is no route at a.shop.example.com to replace" { |
| 674 | t.Fatalf("then the row: %q", got) |
| 675 | } |
| 676 | if got := importRefusal(rec, &row, ReconcileConfig{}, nil, true); got != "the route belongs to app other" { |
| 677 | t.Fatalf("app before account: %q", got) |
| 678 | } |
| 679 | ok := addrRow(rec, "abc") |
| 680 | if got := importRefusal(rec, &ok, ReconcileConfig{}, nil, true); got != "" { |
| 681 | t.Fatalf("the record's own row: %q", got) |
| 682 | } |
| 683 | } |
| 684 | |
amadan