carlos / platform Public

provision, activator: an orphaned import database is decided by its seed marker (Plan 7 Task 5 fix round 2)

With the import's row gone, the rollback now decides its local database
from the file, whoever holds the host's record: under the fence it is
removed only when its <db>.seed marker names this journal's seed and the
host still has no row. A marker that is absent or names another seed
leaves the file (logged); a marker that cannot be read, or a row that
appears, lifts the fence and defers. Step 6's record test now governs the
replica delete only.

- activator.ReadSeedMarker reads the marker through instancefs (absent,
  symlinked, not regular or oversized names no seed; other failures are
  errors); PurgeSeedLocal shares its path check.
- SeedHooks gains LocalSeedID; the rollback's hooks are all six or none.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude agent, operated by paul@keymail.dev aba67ca30db8f4c496ee78d1a0988d9a41d6ad45 parent ae6d2c7
5 files changed, +258 −85
  • internal/activator/seedplace.go +40 −7
  • internal/activator/seedpurge_linux_test.go +53 −0
  • internal/provision/seedreconcile.go +5 −1
  • internal/provision/seedrollback.go +46 −28
  • internal/provision/seedrollback_test.go +114 −49

CI passed — run details

diff --git a/internal/activator/seedplace.go b/internal/activator/seedplace.go
index 5ef20a9cb..0fdc1343b 100644
--- a/internal/activator/seedplace.go
+++ b/internal/activator/seedplace.go
@@ -228,17 +228,50 @@ func seedTempName(id string) string { return ".seed-" + id + ".tmp" }
// the host is fenced and stopped. A directory that is not there holds
// nothing, so a second run, or one for an instance never woken, is done.
func PurgeSeedLocal(dbPath string) error {
- if !filepath.IsAbs(dbPath) || filepath.Clean(dbPath) != dbPath {
- return fmt.Errorf("activator: %q is not a clean absolute database path", dbPath)
- }
- dir, base := filepath.Split(dbPath)
- if base == "" {
- return fmt.Errorf("activator: %q names no database", dbPath)
+ dir, base, err := splitDBPath(dbPath)
+ if err != nil {
+ return err
}
if _, err := os.Lstat(dir); errors.Is(err, os.ErrNotExist) {
return nil
}
- return purgeSeedTarget(filepath.Clean(dir), base)
+ return purgeSeedTarget(dir, base)
+}
+
+// ReadSeedMarker is the seed id the <db>.seed marker beside dbPath names,
+// read through instancefs: the evidence that the database there was placed
+// for that seed (spec §3.4 step 3). No marker, or one that is a symlink,
+// not a regular file or oversized, is "" (it names no seed); so is a
+// directory that is not there. Any other failure to read it is an error:
+// the caller cannot tell.
+func ReadSeedMarker(dbPath string) (string, error) {
+ dir, base, err := splitDBPath(dbPath)
+ if err != nil {
+ return "", err
+ }
+ if _, err := os.Lstat(dir); errors.Is(err, os.ErrNotExist) {
+ return "", nil
+ }
+ m, err := instancefs.ReadBeneathLimit(dir, base+SeedMarkerSuffix, seedMarkerMax)
+ if err != nil {
+ if notAPlacedFile(err) {
+ return "", nil
+ }
+ return "", err
+ }
+ return strings.TrimSpace(string(m)), nil
+}
+
+// splitDBPath is a clean absolute database path's directory and file name.
+func splitDBPath(dbPath string) (dir, base string, err error) {
+ if !filepath.IsAbs(dbPath) || filepath.Clean(dbPath) != dbPath {
+ return "", "", fmt.Errorf("activator: %q is not a clean absolute database path", dbPath)
+ }
+ dir, base = filepath.Split(dbPath)
+ if base == "" {
+ return "", "", fmt.Errorf("activator: %q names no database", dbPath)
+ }
+ return filepath.Clean(dir), base, nil
}
// purgeSeedTarget removes everything at and beside dir/base that could
diff --git a/internal/activator/seedpurge_linux_test.go b/internal/activator/seedpurge_linux_test.go
index 23d2f8d72..28913a3bb 100644
--- a/internal/activator/seedpurge_linux_test.go
+++ b/internal/activator/seedpurge_linux_test.go
@@ -118,3 +118,56 @@ func TestAPlainWakeAfterPurgeSeedLocalFindsNoDatabase(t *testing.T) {
}
}
}
+
+// Fix round 2: the marker beside a database is read through instancefs.
+// It names a seed, or names none (absent, a symlink, a directory, no
+// instance directory), or cannot be read (an error).
+func TestReadSeedMarker(t *testing.T) {
+ dir := filepath.Join(t.TempDir(), "instance")
+ if err := os.MkdirAll(dir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ db := filepath.Join(dir, "a.shop.example.com.db")
+ marker := db + SeedMarkerSuffix
+ check := func(name, want string, wantErr bool) {
+ t.Helper()
+ got, err := ReadSeedMarker(db)
+ if got != want || (err != nil) != wantErr {
+ t.Fatalf("%s: %q, %v; want %q, error %v", name, got, err, want, wantErr)
+ }
+ }
+ check("no marker", "", false)
+ if err := os.WriteFile(marker, []byte("sd_aaaaaaaaaaaaaaaaaaaaaaaaaa\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ check("a marker", "sd_aaaaaaaaaaaaaaaaaaaaaaaaaa", false)
+ os.Remove(marker)
+ target := filepath.Join(t.TempDir(), "elsewhere")
+ os.WriteFile(target, []byte("sd_bbbbbbbbbbbbbbbbbbbbbbbbbb"), 0o600)
+ if err := os.Symlink(target, marker); err != nil {
+ t.Fatal(err)
+ }
+ check("a symlinked marker", "", false)
+ os.Remove(marker)
+ if err := os.Mkdir(marker, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ check("a directory at the marker", "", false)
+ os.Remove(marker)
+ if got, err := ReadSeedMarker(filepath.Join(t.TempDir(), "gone", "a.db")); got != "" || err != nil {
+ t.Fatalf("no instance directory: %q %v", got, err)
+ }
+ if os.Getuid() != 0 {
+ if err := os.WriteFile(marker, []byte("sd_aaaaaaaaaaaaaaaaaaaaaaaaaa"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chmod(dir, 0o000); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.Chmod(dir, 0o700) })
+ check("an unreadable directory", "", true)
+ }
+ if _, err := ReadSeedMarker("relative.db"); err == nil {
+ t.Fatal("a relative path read")
+ }
+}
diff --git a/internal/provision/seedreconcile.go b/internal/provision/seedreconcile.go
index 60b348ce6..3e87583e7 100644
--- a/internal/provision/seedreconcile.go
+++ b/internal/provision/seedreconcile.go
@@ -28,7 +28,7 @@ type SeedHooks struct {
// intent alone decides.
NestedOK func() error
- // The rollback's box effects (spec §5.6, seedrollback.go). All five or
+ // The rollback's box effects (spec §5.6, seedrollback.go). All six or
// none: a box without them leaves every abandoned import journaled,
// gated and untouched.
//
@@ -48,6 +48,10 @@ type SeedHooks struct {
DeleteReplica func(ctx context.Context, host string) error
// NextFenceGen mints a fence generation no fence on this box holds.
NextFenceGen func() uint64
+ // LocalSeedID is the seed id the <db>.seed marker beside dbPath names
+ // ("" for none or no marker), read through instancefs. An error is a
+ // marker that could not be read.
+ LocalSeedID func(dbPath string) (string, error)
}
type seedVerdict int
diff --git a/internal/provision/seedrollback.go b/internal/provision/seedrollback.go
index 99741b531..fa376d87d 100644
--- a/internal/provision/seedrollback.go
+++ b/internal/provision/seedrollback.go
@@ -30,8 +30,9 @@ import (
// journal row came back from a snapshot, and the rollback holds there,
// touching nothing, until an operator looks. A host with no row at all is
// rolled back without a restore: whatever removed the import's row decided
-// the host should not be served. Its local data at the host's default path
-// is still removed when step 6's test says it is the import's.
+// the host should not be served. Its local database at the host's default
+// path is still removed when the database's <db>.seed marker names this
+// seed and the host still has no row.
//
// rec is the pass's view of the journal's record (nil when gone). It chose
// the rollback; the destructive cleanup reads the record again (step 6).
@@ -41,7 +42,8 @@ func rollbackImport(ctx context.Context, s release.Store, reg *registry.Registry
log.Printf("provision: dry-run: would roll back %s's import %s", j.Host, j.SeedID)
return
}
- if h == nil || h.FenceForRollback == nil || h.LiftFence == nil || h.RemoveLocal == nil || h.DeleteReplica == nil || h.NextFenceGen == nil {
+ if h == nil || h.FenceForRollback == nil || h.LiftFence == nil || h.RemoveLocal == nil || h.DeleteReplica == nil || h.NextFenceGen == nil ||
+ h.LocalSeedID == nil {
log.Printf("provision: %s: import %s cannot roll back on this box: the activator is not wired; it stays journaled and gated", j.Host, j.SeedID)
return
}
@@ -126,25 +128,8 @@ func rollBackRoute(ctx context.Context, s release.Store, reg *registry.Registry,
}
switch {
case row == nil:
- // Something removed the import's row, but not its local data: a
- // later plain instance at the host would open that database as a
- // warm cache and replicate it as its own. Whether the data at the
- // host's default path is the import's is step 6's question.
- owned, why, err := importOwnsHost(ctx, s, j)
- switch {
- case err != nil:
- h.LiftFence(j.Host, j.FenceGen)
- log.Printf("provision: %s: import %s rollback deferred: whose local data is at the host cannot be told: %v", j.Host, j.SeedID, err)
+ if !removeOrphanedLocal(reg, cfg, j) {
return j, false
- case !owned:
- log.Printf("provision: %s: import %s has no route on this box, and its local data is left alone: %s", j.Host, j.SeedID, why)
- default:
- local := registry.Route{Host: j.Host, Kind: "instance", DBPath: NestedDBPath(cfg.DataDir, j.Host)}
- if err := h.RemoveLocal(local); err != nil {
- log.Printf("provision: %s: import %s: remove its local data: %v", j.Host, j.SeedID, err)
- return j, false
- }
- log.Printf("provision: %s: import %s has no route on this box: its local data is removed and no route is put back", j.Host, j.SeedID)
}
case !importRowOf(cfg, *row, j) || row.ImportGate != registry.ImportGateClosed:
h.LiftFence(j.Host, j.FenceGen)
@@ -225,15 +210,48 @@ func cleanUpImport(ctx context.Context, s release.Store, reg *registry.Registry,
}
}
-// importOwnsHost is replicaIsTheImports on a fresh read of the journal's
-// record: whether what the box holds for the host by host name alone (its
-// replica, and its local data at the default path) is still the import's.
-func importOwnsHost(ctx context.Context, s release.Store, j registry.ImportJournal) (owned bool, why string, err error) {
- rec, err := journalRecordByKey(ctx, s, j)
+// removeOrphanedLocal is step 3 for an import whose row something removed,
+// run under the fence. Its local database is still at the host's default
+// nested path (routeFor's), where a later instance at the host would open
+// it as a warm cache and replicate it as its own. It is decided from the
+// file, whoever holds the host's record: while the journal row exists no
+// other row can be minted at the host, so under the fence nothing else
+// has that file open. The database is removed only when its <db>.seed
+// marker names this seed and the host still has no row. A marker that is
+// absent or names another seed does not prove the data is the import's,
+// so the file is left. ok false defers the rollback, with the fence
+// lifted.
+func removeOrphanedLocal(reg *registry.Registry, cfg ReconcileConfig, j registry.ImportJournal) (ok bool) {
+ h := cfg.Seeds
+ db := NestedDBPath(cfg.DataDir, j.Host)
+ id, err := h.LocalSeedID(db)
+ if err != nil {
+ h.LiftFence(j.Host, j.FenceGen)
+ log.Printf("provision: %s: import %s rollback deferred: the seed marker beside %s cannot be read: %v", j.Host, j.SeedID, db, err)
+ return false
+ }
+ if id != j.SeedID {
+ log.Printf("provision: %s: import %s has no route on this box; the database at %s is left: its seed marker names %q, not this seed", j.Host, j.SeedID, db, id)
+ return true
+ }
+ row, err := reg.Get(j.Host)
if err != nil {
- return false, "", fmt.Errorf("its record cannot be read: %w", err)
+ h.LiftFence(j.Host, j.FenceGen)
+ log.Printf("provision: %s: import %s rollback deferred: the route could not be read: %v", j.Host, j.SeedID, err)
+ return false
+ }
+ if row != nil {
+ // Not expected while journaled. The next pass judges the row.
+ h.LiftFence(j.Host, j.FenceGen)
+ log.Printf("provision: %s: import %s: a route appeared at the host, so its local database is left and the rollback waits", j.Host, j.SeedID)
+ return false
+ }
+ if err := h.RemoveLocal(registry.Route{Host: j.Host, Kind: "instance", DBPath: db}); err != nil {
+ log.Printf("provision: %s: import %s: remove its local data: %v", j.Host, j.SeedID, err)
+ return false
}
- return replicaIsTheImports(ctx, s, j, rec)
+ log.Printf("provision: %s: import %s has no route on this box: its local data is removed and no route is put back", j.Host, j.SeedID)
+ return true
}
// replicaIsTheImports decides step 6's replica delete. The host's replica
diff --git a/internal/provision/seedrollback_test.go b/internal/provision/seedrollback_test.go
index 41b41294d..0f61fcfee 100644
--- a/internal/provision/seedrollback_test.go
+++ b/internal/provision/seedrollback_test.go
@@ -22,6 +22,7 @@ type rollbackRecorder struct {
proceed bool
fenceErr error
gen uint64
+ marker string // the seed id the local database's marker names
}
func (r *rollbackRecorder) hooks(h *SeedHooks) {
@@ -33,6 +34,7 @@ func (r *rollbackRecorder) hooks(h *SeedHooks) {
h.LiftFence = func(host string, gen uint64) { r.steps = append(r.steps, "lift") }
h.RemoveLocal = func(rt registry.Route) error { r.steps = append(r.steps, "remove-local:"+rt.Kind); return nil }
h.DeleteReplica = func(_ context.Context, host string) error { r.steps = append(r.steps, "delete-replica"); return nil }
+ h.LocalSeedID = func(string) (string, error) { return r.marker, nil }
}
func TestRollbackAfterAbortRestoresTheRowExactly(t *testing.T) {
@@ -266,6 +268,8 @@ type rollbackBox struct {
fault string // the step that fails, once
crash bool // the fault kills the process (a panic) rather than returning an error
noRoute bool // the import's row was removed: no route comes back
+ marker string // the seed id the local database's <db>.seed names
+ markErr error // reading that marker fails
}
func newRollbackBox(t *testing.T, reg *registry.Registry, original registry.Route) *rollbackBox {
@@ -374,6 +378,19 @@ func (b *rollbackBox) hooks(h *SeedHooks) {
b.local = false
return b.trip("remove-local-done")
}
+ h.LocalSeedID = func(dbPath string) (string, error) {
+ b.steps = append(b.steps, "read-marker")
+ if dbPath != NestedDBPath("/data", b.original.Host) {
+ b.t.Errorf("read the marker beside %s", dbPath)
+ }
+ if b.markErr != nil {
+ return "", b.markErr
+ }
+ if !b.local {
+ return "", nil
+ }
+ return b.marker, nil
+ }
h.DeleteReplica = func(_ context.Context, host string) error {
b.steps = append(b.steps, "delete-replica")
b.wantRestored("deleted the replica")
@@ -438,6 +455,7 @@ func abortedImport(t *testing.T) *rollbackScene {
rec := importRecord(t, s, sqid)
reg, path := regAt(t, addrRow(rec, sqid))
box := newRollbackBox(t, reg, *mustGet(t, reg, rec.Host))
+ box.marker = rec.Seed.ID // the verify wake placed this seed
cfg := importCfg(t, nil)
box.hooks(cfg.Seeds)
data := cfg.Seeds.Data
@@ -782,8 +800,8 @@ func TestRollbackOfAStaleJournalTouchesNothingTheNewInstanceOwns(t *testing.T) {
// The rollback tolerates a missing import row (the Task 4 review's carry):
// something removed it, so no route is put back, and the cleanup carries
-// on. Its local data is still removed under the fence (fix round 1, I1):
-// the record names this seed, aborted, and no other record holds the host.
+// on. Its local data is still removed under the fence (fix rounds 1 and 2):
+// the database's marker names this seed and the host has no row.
func TestRollbackOfAMissingRowSkipsTheRestore(t *testing.T) {
x := abortedImport(t)
if err := x.reg.Delete(x.rec.Host); err != nil {
@@ -794,7 +812,7 @@ func TestRollbackOfAMissingRowSkipsTheRestore(t *testing.T) {
if rt, _ := x.reg.Get(x.rec.Host); rt != nil {
t.Fatalf("a route came back: %+v", rt)
}
- want := []string{"gen", "fence", "remove-local", "lift", "delete-snapshot", "delete-replica"}
+ want := []string{"gen", "fence", "read-marker", "remove-local", "lift", "delete-snapshot", "delete-replica"}
if !reflect.DeepEqual(x.box.steps, want) {
t.Fatalf("steps = %q, want %q", x.box.steps, want)
}
@@ -940,51 +958,71 @@ func TestRollbackCleanupOfAnInvalidSeedIDDeletesNothing(t *testing.T) {
}
}
-// Fix round 1, I1: a missing row whose host another record now holds. The
-// data at the host's default path may be that record's, so it is left,
-// and logged; the replica is left for the same reason.
-func TestRollbackOfAMissingRowLeavesAnotherRecordsLocalData(t *testing.T) {
- x := abortedImport(t)
- if err := x.reg.Delete(x.rec.Host); err != nil {
- t.Fatal(err)
- }
- x.box.noRoute = true
- other := shopRecord(x.rec.Account)
- other.App = "other"
- if err := CreateRecord(ctx, x.raw, other); err != nil {
- t.Fatal(err)
- }
- logged := captureLog(t)
- x.pass(t)
- if count(x.box.steps, "remove-local") != 0 || !x.box.local || !x.box.replica {
- t.Fatalf("steps %q local %v replica %v", x.box.steps, x.box.local, x.box.replica)
- }
- if !strings.Contains(logged.String(), "its local data is left alone: the record "+RecordKey(other.Account, other.App, other.Host)+" holds the host") {
- t.Fatalf("not logged:\n%s", logged)
- }
- if x.journalState(t) != "" || len(x.box.fenced) != 0 {
- t.Fatalf("journal %q fenced %v", x.journalState(t), x.box.fenced)
- }
-}
-
-// Fix round 1, I1: a missing row whose record cannot be read under the
-// fence. Nothing is removed, the fence is lifted, and the rollback waits
-// at rolling-back.
-func TestRollbackOfAMissingRowDefersWhenItCannotTell(t *testing.T) {
- x := abortedImport(t)
- if err := x.reg.Delete(x.rec.Host); err != nil {
- t.Fatal(err)
- }
- x.box.noRoute = true
- // The pass's own read works; the one under the fence fails.
- x.store = &failNthGet{Store: x.store, key: RecordKey(x.rec.Account, x.rec.App, x.rec.Host), n: 2}
- x.pass(t)
- want := []string{"gen", "fence", "lift"}
- if !reflect.DeepEqual(x.box.steps, want) || !x.box.local || len(x.box.fenced) != 0 {
- t.Fatalf("steps %q (want %q) local %v fenced %v", x.box.steps, want, x.box.local, x.box.fenced)
- }
- if st := x.journalState(t); st != registry.JournalRollingBack {
- t.Fatalf("journal = %q", st)
+// Fix round 2: with the import's row gone, its local database is decided
+// from the file. Removed when its marker names this seed (whoever holds
+// the host's record, here another app's); left when the marker names
+// another seed or there is none; and a marker that cannot be read lifts
+// the fence and defers. The replica still follows step 6's record test:
+// another record holds the host, so it is kept.
+func TestRollbackOfAMissingRowDecidesTheLocalDataFromItsMarker(t *testing.T) {
+ otherSeed, _ := seeds.NewID()
+ for _, c := range []struct {
+ name string
+ marker func(x *rollbackScene) string
+ markErr error
+ removed bool
+ defers bool
+ logged string
+ }{
+ {name: "the marker names this seed", marker: func(x *rollbackScene) string { return x.rec.Seed.ID }, removed: true,
+ logged: "its local data is removed"},
+ {name: "the marker names another seed", marker: func(*rollbackScene) string { return otherSeed },
+ logged: "its seed marker names \"" + otherSeed + "\", not this seed"},
+ {name: "no marker", marker: func(*rollbackScene) string { return "" },
+ logged: "its seed marker names \"\", not this seed"},
+ {name: "the marker cannot be read", markErr: errors.New("permission denied"), defers: true,
+ logged: "the seed marker beside"},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ x := abortedImport(t)
+ if err := x.reg.Delete(x.rec.Host); err != nil {
+ t.Fatal(err)
+ }
+ x.box.noRoute = true
+ if c.marker != nil {
+ x.box.marker = c.marker(x)
+ }
+ x.box.markErr = c.markErr
+ other := shopRecord(x.rec.Account)
+ other.App = "other"
+ if err := CreateRecord(ctx, x.raw, other); err != nil {
+ t.Fatal(err)
+ }
+ logged := captureLog(t)
+ x.pass(t)
+ if !strings.Contains(logged.String(), c.logged) {
+ t.Fatalf("not logged %q:\n%s", c.logged, logged)
+ }
+ if x.box.local == c.removed || (count(x.box.steps, "remove-local") == 1) != c.removed {
+ t.Fatalf("local %v steps %q; want removed %v", x.box.local, x.box.steps, c.removed)
+ }
+ if len(x.box.fenced) != 0 {
+ t.Fatalf("a fence was left: %v", x.box.fenced)
+ }
+ if c.defers {
+ if st := x.journalState(t); st != registry.JournalRollingBack {
+ t.Fatalf("journal = %q, want the rollback waiting at rolling-back", st)
+ }
+ if want := []string{"gen", "fence", "read-marker", "lift"}; !reflect.DeepEqual(x.box.steps, want) {
+ t.Fatalf("steps = %q, want %q", x.box.steps, want)
+ }
+ return
+ }
+ if x.journalState(t) != "" || !x.box.replica {
+ t.Fatalf("journal %q replica %v: the cleanup did not finish, or deleted another record's replica",
+ x.journalState(t), x.box.replica)
+ }
+ })
}
}
@@ -1004,7 +1042,8 @@ func TestRollbackLiftsTheFenceWhenTheFenceHookFails(t *testing.T) {
}
// Fix round 1, I1, the provision half of the end-to-end check: with the
-// import's row gone, the local removal is aimed at the host's default
+// import's row gone (and its marker naming this seed), the local removal
+// is aimed at the host's default
// nested database path, which is exactly where a plain instance created at
// the host afterwards is minted. (The activator half, seedpurge_linux_test.go,
// shows a plain wake at that path finds no file once PurgeSeedLocal ran.)
@@ -1015,6 +1054,7 @@ func TestRollbackOfAMissingRowPurgesWhereTheNextInstanceIsMinted(t *testing.T) {
cfg := importCfg(t, nil)
rr := &rollbackRecorder{proceed: true}
rr.hooks(cfg.Seeds)
+ rr.marker = rec.Seed.ID
var purged []string
cfg.Seeds.RemoveLocal = func(rt registry.Route) error { purged = append(purged, rt.DBPath); return nil }
if err := Reconcile(ctx, s, reg, cfg); err != nil {
@@ -1068,3 +1108,28 @@ func TestRollbackLiftsAKeptFenceWhenTheRowIsNoLongerTheImports(t *testing.T) {
t.Fatalf("fenced %v journal %q", x.box.fenced, x.journalState(t))
}
}
+
+// Fix round 2: a row that appears at the host while the marker is read
+// (not expected while journaled) keeps the local database, lifts the
+// fence, and leaves the rollback waiting for the next pass to judge it.
+func TestRollbackOfAMissingRowLeavesTheDataWhenARowAppears(t *testing.T) {
+ x := abortedImport(t)
+ if err := x.reg.Delete(x.rec.Host); err != nil {
+ t.Fatal(err)
+ }
+ x.box.noRoute = true
+ read := x.cfg.Seeds.LocalSeedID
+ x.cfg.Seeds.LocalSeedID = func(dbPath string) (string, error) {
+ if err := x.reg.Upsert(addrRow(x.rec, x.rec.Account)); err != nil {
+ t.Fatal(err)
+ }
+ return read(dbPath)
+ }
+ x.pass(t)
+ if !x.box.local || count(x.box.steps, "remove-local") != 0 || len(x.box.fenced) != 0 {
+ t.Fatalf("local %v steps %q fenced %v", x.box.local, x.box.steps, x.box.fenced)
+ }
+ if st := x.journalState(t); st != registry.JournalRollingBack {
+ t.Fatalf("journal = %q", st)
+ }
+}