| index a95173f35..6751eb2d8 100644 |
| --- a/internal/provision/seedreconcile_test.go |
| +++ b/internal/provision/seedreconcile_test.go |
| @@ -4,6 +4,9 @@ import ( |
| "context" |
| "errors" |
| "os" |
| + "path/filepath" |
| + "reflect" |
| + "strings" |
| "testing" |
| "time" |
| |
| @@ -240,3 +243,246 @@ func TestReconcileNeverRemovesAnAbortedImportsRow(t *testing.T) { |
| t.Fatalf("record = %+v, %v; want rolled-back", got.Seed, err) |
| } |
| } |
| + |
| +// diskBox is the rollback's six box hooks over a real data directory: the |
| +// marker and the database are files, read and removed as the activator's |
| +// ReadSeedMarker and PurgeSeedLocal do (their own tests are the activator's |
| +// seedpurge_linux_test.go). It records the steps in order. |
| +type diskBox struct { |
| + t *testing.T |
| + s release.Store |
| + rec Record |
| + steps []string |
| + proceed bool |
| + gen uint64 |
| + fenced map[string]uint64 |
| + markErr error |
| +} |
| + |
| +func newDiskBox(t *testing.T, s release.Store, rec Record) *diskBox { |
| + return &diskBox{t: t, s: s, rec: rec, proceed: true, fenced: map[string]uint64{}} |
| +} |
| + |
| +func (b *diskBox) hooks(h *SeedHooks) { |
| + h.NextFenceGen = func() (uint64, error) { b.gen++; return b.gen, nil } |
| + h.FenceForRollback = func(_ context.Context, host string, gen uint64) (bool, error) { |
| + b.steps = append(b.steps, "fence") |
| + if !b.proceed { |
| + return false, nil |
| + } |
| + b.fenced[host] = gen |
| + return true, nil |
| + } |
| + h.LiftFence = func(host string, gen uint64) { |
| + b.steps = append(b.steps, "lift") |
| + if b.fenced[host] == gen { |
| + delete(b.fenced, host) |
| + } |
| + } |
| + h.LocalSeedID = func(db string) (string, error) { |
| + b.steps = append(b.steps, "read-marker") |
| + if b.markErr != nil { |
| + return "", b.markErr |
| + } |
| + m, err := os.ReadFile(db + ".seed") |
| + if errors.Is(err, os.ErrNotExist) { |
| + return "", nil |
| + } |
| + return string(m), err |
| + } |
| + h.RemoveLocal = func(rt registry.Route) error { |
| + b.steps = append(b.steps, "remove-local") |
| + if _, ok := b.fenced[rt.Host]; !ok { |
| + b.t.Errorf("removed the local data of %s while it was not fenced and stopped", rt.Host) |
| + } |
| + // Before the rollback: the record is still aborted. |
| + if got, err := GetRecord(ctx, b.s, b.rec.Account, b.rec.App, b.rec.Host); err != nil || got.Seed.State != seeds.StateAborted { |
| + b.t.Errorf("removed the local data with the record at %+v, %v; want aborted", got.Seed, err) |
| + } |
| + for _, p := range []string{rt.DBPath, rt.DBPath + "-wal", rt.DBPath + ".seed"} { |
| + if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) { |
| + return err |
| + } |
| + } |
| + return nil |
| + } |
| + h.DeleteReplica = func(context.Context, string) error { |
| + b.t.Error("the no-journal path deleted a replica") |
| + return nil |
| + } |
| +} |
| + |
| +// placeLocal writes a database and its <db>.seed marker naming id at db, |
| +// as a wake that placed the seed leaves them. |
| +func placeLocal(t *testing.T, db, id string) { |
| + t.Helper() |
| + if err := os.MkdirAll(filepath.Dir(db), 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + for p, body := range map[string]string{db: "the seed's data", db + "-wal": "wal", db + ".seed": id} { |
| + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| +} |
| + |
| +func diskSeedCfg(t *testing.T, data release.Store, box *diskBox) ReconcileConfig { |
| + cfg := nestedSeedCfg(data) |
| + cfg.DataDir = t.TempDir() |
| + box.hooks(cfg.Seeds) |
| + return cfg |
| +} |
| + |
| +func requireSeedState(t *testing.T, s release.Store, rec Record, state string) { |
| + t.Helper() |
| + got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host) |
| + if err != nil || got.Seed == nil || got.Seed.State != state { |
| + t.Fatalf("record = %+v, %v; want %s", got.Seed, err, state) |
| + } |
| +} |
| + |
| +// Final review F2: a --from seed whose publish landed but whose answer was |
| +// lost. A box minted its row and a wake placed its database; then the abort |
| +// won. The row goes first; the pass that finds no row removes the database |
| +// (its marker names this seed) under a fence, before the record is rolled |
| +// back. That pass runs as after an agent restart: nothing in memory, only |
| +// the marker on disk. A plain instance created at the host afterwards is |
| +// minted at the same path and finds no file there. |
| +func TestReconcileAbortedFromSeedRemovesItsLocalDatabase(t *testing.T) { |
| + s, sqid := setup(t) |
| + data := dirStore(t) |
| + rec := seededRecord(t, s, sqid, seeds.KindReplica, seeds.StateReady) |
| + box := newDiskBox(t, s, rec) |
| + cfg := diskSeedCfg(t, data, box) |
| + reg := regWith(t) |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + db := mustGet(t, reg, rec.Host).DBPath |
| + if db != NestedDBPath(cfg.DataDir, rec.Host) { |
| + t.Fatalf("minted at %s", db) |
| + } |
| + placeLocal(t, db, rec.Seed.ID) |
| + if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedAbort(rec.Seed.ID)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + if rt, _ := reg.Get(rec.Host); rt != nil { |
| + t.Fatalf("the aborted seed's row is still up: %+v", rt) |
| + } |
| + requireSeedState(t, s, rec, seeds.StateAborted) |
| + if _, err := os.Stat(db); err != nil { |
| + t.Fatalf("the database went with the row, before the no-row pass: %v", err) |
| + } |
| + |
| + restarted := newDiskBox(t, s, rec) // the agent restarted: no fences, no memory |
| + restarted.hooks(cfg.Seeds) |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + requireSeedState(t, s, rec, seeds.StateRolledBack) |
| + if want := []string{"read-marker", "fence", "read-marker", "remove-local", "lift"}; !reflect.DeepEqual(restarted.steps, want) { |
| + t.Fatalf("steps = %q, want %q", restarted.steps, want) |
| + } |
| + if len(restarted.fenced) != 0 { |
| + t.Fatalf("a fence was left: %v", restarted.fenced) |
| + } |
| + for _, p := range []string{db, db + "-wal", db + ".seed"} { |
| + if _, err := os.Lstat(p); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("%s survived the rollback: %v", p, err) |
| + } |
| + } |
| + |
| + // The console's finish deletes the record; a plain instance follows. |
| + if err := DeleteRecord(ctx, s, rec.Account, rec.App, rec.Host); err != nil { |
| + t.Fatal(err) |
| + } |
| + plain := acmeRecord(sqid) |
| + if err := CreateRecord(ctx, s, plain); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + next := mustGet(t, reg, plain.Host) |
| + if next.SeedSource != "" { |
| + t.Fatalf("the plain instance's row = %+v", next) |
| + } |
| + if _, err := os.Lstat(next.DBPath); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("the plain instance at %s finds a database: %v", next.DBPath, err) |
| + } |
| +} |
| + |
| +// The marker decides, as for an import whose row is gone (Plan 7): a |
| +// marker naming another seed, or none, rolls back with the file left; a |
| +// marker that cannot be read, or a fence that does not proceed, keeps the |
| +// seed aborted with everything in place. |
| +func TestReconcileAbortedFromSeedDecidesItsLocalDataFromTheMarker(t *testing.T) { |
| + other, _ := seeds.NewID() |
| + for _, c := range []struct { |
| + name string |
| + marker string // "" writes no marker; "this" names the seed |
| + markErr error |
| + noFence bool |
| + state string |
| + removed bool |
| + steps []string |
| + logged string |
| + }{ |
| + {name: "this seed", marker: "this", state: seeds.StateRolledBack, removed: true, |
| + steps: []string{"read-marker", "fence", "read-marker", "remove-local", "lift"}, logged: "its local data is removed"}, |
| + {name: "another seed", marker: other, state: seeds.StateRolledBack, |
| + steps: []string{"read-marker"}, logged: "its seed marker names \"" + other + "\", not this seed"}, |
| + {name: "no marker", state: seeds.StateRolledBack, |
| + steps: []string{"read-marker"}, logged: "its seed marker names \"\", not this seed"}, |
| + {name: "unreadable marker", marker: "this", markErr: errors.New("permission denied"), state: seeds.StateAborted, |
| + steps: []string{"read-marker"}, logged: "the seed marker beside"}, |
| + {name: "not fenced", marker: "this", noFence: true, state: seeds.StateAborted, |
| + steps: []string{"read-marker", "fence", "lift"}, logged: "is not fenced and stopped yet"}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + s, sqid := setup(t) |
| + data := dirStore(t) |
| + rec := seededRecord(t, s, sqid, seeds.KindReplica, seeds.StateAborted) |
| + snap := seeds.SnapshotKey(rec.Host, rec.Seed.ID) |
| + if _, err := data.Put(ctx, snap, []byte("db"), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + box := newDiskBox(t, s, rec) |
| + box.markErr, box.proceed = c.markErr, !c.noFence |
| + cfg := diskSeedCfg(t, data, box) |
| + db := NestedDBPath(cfg.DataDir, rec.Host) |
| + switch c.marker { |
| + case "": |
| + placeLocal(t, db, "") |
| + if err := os.Remove(db + ".seed"); err != nil { |
| + t.Fatal(err) |
| + } |
| + case "this": |
| + placeLocal(t, db, rec.Seed.ID) |
| + default: |
| + placeLocal(t, db, c.marker) |
| + } |
| + logged := captureLog(t) |
| + if err := Reconcile(ctx, s, regWith(t), cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + requireSeedState(t, s, rec, c.state) |
| + if !reflect.DeepEqual(box.steps, c.steps) { |
| + t.Fatalf("steps = %q, want %q", box.steps, c.steps) |
| + } |
| + if !strings.Contains(logged.String(), c.logged) { |
| + t.Fatalf("not logged %q:\n%s", c.logged, logged) |
| + } |
| + if _, err := os.Lstat(db); errors.Is(err, os.ErrNotExist) != c.removed { |
| + t.Fatalf("database at %s: %v; want removed %v", db, err, c.removed) |
| + } |
| + _, _, err := data.Get(ctx, snap) |
| + if gone := errors.Is(err, os.ErrNotExist); gone != (c.state == seeds.StateRolledBack) { |
| + t.Fatalf("snapshot: %v; want it deleted only with the rollback", err) |
| + } |
| + }) |
| + } |
| +} |
amadan