carlos / platform Public

host: a journalled archive identity is final and finished from what was published; a listing error is never absence; the archive seal's bound reaches the ledger or the phase stops (Task 5 fix round 1, rulings 4 and 5)

The journal entry now carries the two move-snapshots keys and the
manifest's bytes, written before the first PUT. A replay without the
sealed file finishes that identity: the sealed object published and the
manifest not (a crash between the two PUTs) writes the manifest from the
journal; both published stages them by server-side copy; the sealed object
absent is errArchiveLost and the cut reverts with RevertArchiveRefused,
never a reseal under a new name. snapshotState returns a listing error as
an error, so a transient failure changes nothing. sealArchive returns a
ledger read error instead of advancing on it.

Tests: the between-PUTs crash with and without the file, a listing
failure before a finish and before a revert, the lost identity's revert,
and an injected ledger-read failure after the archive seal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 2 weeks ago b049b85d81cf79d3b9a2d9bce2e011b2c6ddc82d parent 8bb90ff

4 files changed, +275 −66

internal/host/movesarchive.go +88 −49
internal/host/movescut.go +10 −5
internal/host/movescut_test.go +171 −12
internal/host/movespass_test.go +6 −0

diff --git a/internal/host/movesarchive.go b/internal/host/movesarchive.go

index 58b1da6b0..da0acc368 100644

--- a/internal/host/movesarchive.go

+++ b/internal/host/movesarchive.go

@@ -12,6 +12,7 @@ import (

"github.com/carlosframework/platform/internal/lease"

"github.com/carlosframework/platform/internal/moves"

"github.com/carlosframework/platform/internal/moves/archive"

+ "github.com/carlosframework/platform/internal/provision"

)

// The cut's archive (moves spec §7.4 staged, §8.3, codex 19, ruling G14):

@@ -32,29 +33,46 @@ import (

// for the length of one build. The target's T6 does not price this

// source-side headroom; that is a Plan 6 follow-up.

// cutArchiveStep is the archive's identity in the op journal, written

-// before any PUT so a replay publishes the same object (codex 19). Reuse

-// marks the preparing archive the target verified (G14): the seal names

-// its staged versions and nothing is staged again.

+// before any PUT so a replay publishes the same object (codex 19): the

+// archive's name and digests, the two move-snapshots/ keys it is published

+// under, and the manifest's bytes, so a replay that finds the sealed

+// object published and the manifest not can write the manifest without

+// the file (ruling 4). Reuse marks the preparing archive the target

+// verified (G14): the seal names its staged versions and nothing is staged

+// again.

type cutArchiveStep struct {

builtArchive

- Reuse bool `json:"reuse,omitempty"`

+ Reuse bool `json:"reuse,omitempty"`

+ SealedKey string `json:"sealed_key"`

+ ManifestKey string `json:"manifest_key"`

+ Manifest json.RawMessage `json:"manifest"`

}

+// errArchiveLost: the journalled identity can be finished by nobody. The

+// sealed file is gone from this disk and the sealed object was never

+// published, so there is nothing to publish it from, and a new identity

+// is not allowed once one is journalled (ruling 4). The cut reverts.

+var errArchiveLost = errors.New("the cut's archive is lost before it was published")

// cutArchive publishes the cut's one sealed archive (§8.3) and returns the

// seal's files: the sealed object and its manifest by version.

// Its identity is fixed before any PUT (sealing is randomized), in the op

-// journal. The first run reuses the preparing archive when its plaintext

-// digest is the stopped files' and its staged versions are pinned and its

-// sealed file is still on this disk (G14); otherwise it builds and seals

-// a new one. Then, from the journalled identity:

+// journal, and once journalled it is final: a replay finishes that

+// identity or the cut reverts, and never reseals under a new name

+// (ruling 4). The first run reuses the preparing archive when its

+// plaintext digest is the stopped files' and its staged versions are pinned

+// and its sealed file is still on this disk (G14); otherwise it builds and

+// seals a new one. Then, from the journalled identity (publishCutArchive):

// - the sealed file on disk: create-only into move-snapshots/ (an existing

// object is this cut's earlier PUT), then staged into files/ unless it

// is the reused archive, whose staged versions the ledger holds;

-// - no file, but both objects in move-snapshots/: staged into files/ by a

-// guarded server-side copy from there (or, reused, nothing to stage);

-// - neither: nothing was published under that identity, so the archive

-// is rebuilt and resealed and the journal entry overwritten.

+// - no file: what move-snapshots/ holds decides, and only a listing that

+// answered decides it (an error is an error, never absence). The

+// sealed object there and the manifest not: the manifest is written

+// from the journalled bytes. Both there: staged into files/ by a

+// guarded server-side copy (or, reused, nothing to stage). The sealed

+// object absent: errArchiveLost, which is a revert.

// why is a recorded reason the stopped files cannot be archived.

func (c *cutRun) cutArchive(ctx context.Context, plain string) (builtArchive, []moves.SealedObject, string, error) {

@@ -73,22 +91,18 @@ func (c *cutRun) cutArchive(ctx context.Context, plain string) (builtArchive, []

return builtArchive{}, nil, "", err

}

adir := archiveDir(cfg.DataDir, host, id)

- var step cutArchiveStep

- journalled := false

- if raw, ok := j.Steps[cutStepArchive]; ok && json.Unmarshal([]byte(raw), &step) == nil && step.Name == archiveName(step.Sealed) && step.Plain == plain {

- step.dir = adir

- journalled = true

- }

snap := moves.SnapshotsPrefix(host, id)

- if journalled {

- switch onDisk := sealedOnDisk(step.builtArchive); {

- case onDisk:

- case c.snapshotPresent(ctx, vs, snap+step.Name):

- default:

- journalled = false // nothing published under it: rebuild

+ var step cutArchiveStep

+ if raw, ok := j.Steps[cutStepArchive]; ok {

+ // Journalled: final. An entry that cannot be read, or names another

+ // plaintext than the record's, is a corrupt journal, and starting

+ // over on it could publish a second identity: an error.

+ if err := json.Unmarshal([]byte(raw), &step); err != nil || step.Name != archiveName(step.Sealed) || step.Plain != plain ||

+ step.SealedKey != snap+step.Name+".sealed" || step.ManifestKey != snap+step.Name+".json" || len(step.Manifest) == 0 {

+ return builtArchive{}, nil, "", fmt.Errorf("the journalled archive identity of %s is unusable (%v)", id, err)

}

- }

- if !journalled {

+ step.dir = adir

+ } else {

reuse := ledger.ArchivePlain == plain && ledger.ArchiveVersions != nil && ledger.ArchiveVersions.Pinned() && ledger.ArchiveSealed != ""

if reuse {

kept := builtArchive{Name: archiveName(ledger.ArchiveSealed), Sealed: ledger.ArchiveSealed, Plain: plain, dir: adir}

@@ -115,6 +129,11 @@ func (c *cutRun) cutArchive(ctx context.Context, plain string) (builtArchive, []

}

step = cutArchiveStep{builtArchive: a}

}

+ manifest, err := os.ReadFile(step.manifestPath())

+ if err != nil {

+ return builtArchive{}, nil, "", err

+ }

+ step.SealedKey, step.ManifestKey, step.Manifest = snap+step.Name+".sealed", snap+step.Name+".json", manifest

body, err := json.Marshal(step)

if err != nil {

return builtArchive{}, nil, "", err

@@ -129,23 +148,26 @@ func (c *cutRun) cutArchive(ctx context.Context, plain string) (builtArchive, []

}

a := step.builtArchive

files, err := c.publishCutArchive(ctx, own, vs, step, ledger)

+ if errors.Is(err, errArchiveLost) {

+ log.Printf("moves: %s: %v; the cut reverts", host, err)

+ return builtArchive{}, nil, provision.RevertArchiveRefused, nil

+ }

return a, files, "", err

}

-// publishCutArchive is cutArchive's PUTs from a journalled identity.

+// publishCutArchive is cutArchive's PUTs from a journalled identity. The

+// manifest it stages is the journalled one: the file on disk, when there

+// is one, was written by the same build, and the journal is what a replay

+// without the file has.

func (c *cutRun) publishCutArchive(ctx context.Context, own lease.MirrorStore, vs lease.VersionedStore, step cutArchiveStep, ledger moveLedger) ([]moves.SealedObject, error) {

cfg := c.cfg()

host, id := c.rec.Host, c.g.ID

a := step.builtArchive

snap := moves.SnapshotsPrefix(host, id)

- sealedSnap, manifestSnap := snap+a.Name+".sealed", snap+a.Name+".json"

+ sealedSnap, manifestSnap := step.SealedKey, step.ManifestKey

+ manifest := []byte(step.Manifest)

onDisk := sealedOnDisk(a)

- var manifest []byte

if onDisk {

- var err error

- if manifest, err = os.ReadFile(a.manifestPath()); err != nil {

- return nil, err

- }

fp, ok := vs.(lease.FilePutIfAbsent)

if !ok {

return nil, errors.New("this box cannot write the archive create-only")

@@ -153,25 +175,45 @@ func (c *cutRun) publishCutArchive(ctx context.Context, own lease.MirrorStore, v

if _, err := fp.PutFileIfAbsent(ctx, cfg.OwnBucket, sealedSnap, a.sealedPath()); err != nil && !errors.Is(err, lease.ErrExists) {

return nil, fmt.Errorf("snapshot the sealed archive: %w", err)

}

+ if err := c.fault("archive-sealed-snapshot"); err != nil {

+ return nil, err

+ }

if _, err := vs.PutIfAbsent(ctx, cfg.OwnBucket, manifestSnap, manifest); err != nil && !errors.Is(err, lease.ErrExists) {

return nil, fmt.Errorf("snapshot the archive manifest: %w", err)

}

if err := c.fault("archive-own-put"); err != nil {

return nil, err

}

+ } else {

+ // No file: only what was published under the identity can finish

+ // it. A listing that failed decides nothing (ruling 4).

+ sealedThere, _, err := c.snapshotState(ctx, vs, sealedSnap)

+ if err != nil {

+ return nil, err

+ }

+ if !sealedThere {

+ return nil, fmt.Errorf("%w: %s is not on this disk and not in %s", errArchiveLost, a.Name, snap)

+ }

+ manifestThere, _, err := c.snapshotState(ctx, vs, manifestSnap)

+ if err != nil {

+ return nil, err

+ }

+ if !manifestThere {

+ // A crash between the two PUTs: the manifest from the journal.

+ if _, err := vs.PutIfAbsent(ctx, cfg.OwnBucket, manifestSnap, manifest); err != nil && !errors.Is(err, lease.ErrExists) {

+ return nil, fmt.Errorf("snapshot the archive manifest from the journal: %w", err)

+ }

+ log.Printf("moves: %s: the manifest of %s was written from the journalled identity", host, a.Name)

+ }

}

if step.Reuse {

ref := ledger.ArchiveVersions

if ref == nil || !ref.Pinned() || ledger.ArchiveSealed != a.Sealed {

return nil, fmt.Errorf("the ledger no longer names the archive %s", a.Name)

}

- msize := int64(len(manifest))

- if !onDisk {

- msize = c.snapshotSize(ctx, vs, manifestSnap)

- }

return []moves.SealedObject{

{Key: ref.SealedObjectKey, VersionID: ref.SealedVersion, Size: a.SealedSize},

- {Key: ref.ManifestKey, VersionID: ref.ManifestVersion, Size: msize},

+ {Key: ref.ManifestKey, VersionID: ref.ManifestVersion, Size: int64(len(manifest))},

}, nil

}

prefix := moves.FilesPrefix(host, id)

@@ -244,24 +286,21 @@ func keptManifest(a builtArchive) (archive.Manifest, bool) {

return m, true

}

-// snapshotPresent reports whether move-snapshots/ holds both objects of the

-// archive named base (key without suffix), current and not deleted.

-func (c *cutRun) snapshotPresent(ctx context.Context, vs lease.VersionedStore, base string) bool {

- return c.snapshotSize(ctx, vs, base+".sealed") >= 0 && c.snapshotSize(ctx, vs, base+".json") >= 0

-// snapshotSize is the current size of key in this box's bucket, or -1.

-func (c *cutRun) snapshotSize(ctx context.Context, vs lease.VersionedStore, key string) int64 {

+// snapshotState reports whether key holds a current object (not a delete

+// marker) in this box's bucket, and its size. A listing that fails is an

+// error, never absence (ruling 4): absence lets a replay decide the

+// identity was never published.

+func (c *cutRun) snapshotState(ctx context.Context, vs lease.VersionedStore, key string) (present bool, size int64, err error) {

vers, err := vs.ListVersions(ctx, c.cfg().OwnBucket, key)

if err != nil {

- return -1

+ return false, 0, fmt.Errorf("list %s: %w", key, err)

}

for _, v := range vers {

if v.Key == key && v.IsLatest && !v.DeleteMarker {

- return v.Size

+ return true, v.Size, nil

}

- return -1

+ return false, 0, nil

}

// sealNamesArchive reports whether the seal's files are the archive with

diff --git a/internal/host/movescut.go b/internal/host/movescut.go

index d4373eea4..9050524e2 100644

--- a/internal/host/movescut.go

+++ b/internal/host/movescut.go

@@ -1091,12 +1091,17 @@ func (c *cutRun) sealArchive(ctx context.Context) (builtArchive, bool, error) {

if err != nil {

return builtArchive{}, false, err

}

+ // G7: the bound in the ledger as well as the seal. A ledger that cannot

+ // be read or written stops the phase here; the replay seals again and

+ // writes it (ruling 5).

ledgerKey := moves.LedgerKey(c.rec.Host, c.g.ID)

- if ledger, err := readMoveLedger(ctx, own, cfg.OwnBucket, ledgerKey); err == nil {

- ledger.LeaseUntil = seal.LeaseUntil

- if err := writeMoveLedger(ctx, own, cfg.OwnBucket, ledgerKey, ledger); err != nil {

- return builtArchive{}, false, err

- }

+ ledger, err := readMoveLedger(ctx, own, cfg.OwnBucket, ledgerKey)

+ if err != nil {

+ return builtArchive{}, false, err

+ }

+ ledger.LeaseUntil = seal.LeaseUntil

+ if err := writeMoveLedger(ctx, own, cfg.OwnBucket, ledgerKey, ledger); err != nil {

+ return builtArchive{}, false, err

}

return a, true, nil

}

diff --git a/internal/host/movescut_test.go b/internal/host/movescut_test.go

index 0a72af438..ecd90aeb2 100644

--- a/internal/host/movescut_test.go

+++ b/internal/host/movescut_test.go

@@ -17,6 +17,7 @@ import (

"github.com/carlosframework/platform/internal/lease"

"github.com/carlosframework/platform/internal/lease/leasetest"

"github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/moves/archive"

"github.com/carlosframework/platform/internal/provision"

"github.com/carlosframework/platform/internal/registry"

"github.com/carlosframework/platform/internal/release"

@@ -695,12 +696,15 @@ func snapshotSealed(h *cutHarness) []string {

}

// codex 19, with the real randomized sealer: the archive's identity is

-// journalled before any PUT, and every replay publishes that same key. A

-// replay with the sealed file gone and nothing published reseals under a

-// new key, and the record names the new one; one with the file gone and

-// the snapshot published reuses it.

+// journalled before any PUT, and every replay publishes that same key,

+// whatever it crashed after: the journal entry, the first snapshot PUT,

+// the second, the first copy. Once journalled the identity is final

+// (fix round 1, ruling 4): a replay with the sealed file gone finishes it

+// from what was published (the manifest from the journalled bytes if only

+// the sealed object landed), and one that finds nothing published cannot,

+// so the cut reverts rather than reseal under a new name.

func TestCutArchiveIsCreateOnlyAndReused(t *testing.T) {

- for _, point := range []string{"archive-journaled", "archive-own-put", "archive-first-copy"} {

+ for _, point := range []string{"archive-journaled", "archive-sealed-snapshot", "archive-own-put", "archive-first-copy"} {

t.Run("crash "+point, func(t *testing.T) {

h := newCutHarness(t)

h.ready(t)

@@ -742,15 +746,59 @@ func TestCutArchiveIsCreateOnlyAndReused(t *testing.T) {

}

h.pass(t)

m := h.move(t)

- if m.State != provision.MoveLanding || m.Source.Files.Sealed == first.Sealed {

- t.Fatalf("not resealed: %+v files %+v", m, m.Source.Files)

+ if m.State != provision.MoveReverting || m.Reason != provision.RevertArchiveRefused {

+ t.Fatalf("an identity nobody can finish did not revert: %+v", m)

}

- second := h.journalArchive(t)

- if second.Sealed != m.Source.Files.Sealed || !sealNamesArchive(h.seal(t), second.Sealed) {

- t.Fatalf("journal %s, record %s", second.Sealed, m.Source.Files.Sealed)

+ if again := h.journalArchive(t); again.Sealed != first.Sealed {

+ t.Fatalf("the journalled identity changed from %s to %s", first.Name, again.Name)

}

- if got := snapshotSealed(h); len(got) != 1 || !strings.HasSuffix(got[0], second.Name+".sealed") {

- t.Fatalf("snapshots %v", got)

+ if got := snapshotSealed(h); len(got) != 0 {

+ t.Fatalf("snapshots %v, want none", got)

+ }

+ if h.seal(t).HasFiles() {

+ t.Fatal("the seal names files")

+ }

+ })

+ t.Run("file gone, sealed snapshot only", func(t *testing.T) {

+ h := newCutHarness(t)

+ h.ready(t)

+ h.writeOnStop(t)

+ h.crashAt = "archive-sealed-snapshot"

+ h.cut(t)

+ h.crashPass(t)

+ first := h.journalArchive(t)

+ snap := moves.SnapshotsPrefix(p1Host, p1ID)

+ if _, ok := h.f.buckets["data-ie"][snap+first.Name+".sealed"]; !ok {

+ t.Fatal("fixture: the sealed object did not land")

+ }

+ if _, ok := h.f.buckets["data-ie"][snap+first.Name+".json"]; ok {

+ t.Fatal("fixture: the manifest landed too")

+ }

+ if err := os.RemoveAll(archiveDir(h.cfg.DataDir, p1Host, p1ID)); err != nil {

+ t.Fatal(err)

+ }

+ h.freshWorker()

+ h.pass(t)

+ m := h.move(t)

+ if m.State != provision.MoveLanding || m.Source.Files.Sealed != first.Sealed || !sealNamesArchive(h.seal(t), first.Sealed) {

+ t.Fatalf("the identity was not finished: %+v files %+v", m, m.Source.Files)

+ }

+ manifest, ok := h.f.buckets["data-ie"][snap+first.Name+".json"]

+ if !ok {

+ t.Fatal("the manifest was not written from the journal")

+ }

+ var man archive.Manifest

+ if err := json.Unmarshal([]byte(manifest.body), &man); err != nil || man.Sealed != first.Sealed || man.Plain != first.Plain {

+ t.Fatalf("the written manifest: %v %+v", err, man)

+ }

+ copied := 0

+ for _, c := range h.f.stagingCalls() {

+ if c.Op == "copy" && strings.HasPrefix(c.Key, moves.FilesPrefix(p1Host, p1ID)+first.Name) {

+ copied++

+ }

+ if copied != 2 || len(snapshotSealed(h)) != 1 {

+ t.Fatalf("%d copies from the snapshot, %d sealed snapshots", copied, len(snapshotSealed(h)))

}

})

t.Run("file gone, snapshot published", func(t *testing.T) {

@@ -1216,3 +1264,114 @@ func TestRegularFileDistinguishesAbsence(t *testing.T) {

t.Fatalf("a directory: %v %v", ok, err)

}

+// Ruling 4: a listing error is an error, never absence. With the sealed

+// file gone, a replay whose ListVersions fails changes nothing: the record

+// stays staged, the journalled identity stands, nothing is rebuilt and

+// nothing reverts. The next replay, listing again, finishes the identity

+// when something was published, and reverts when nothing was.

+func TestArchiveListingFailureIsNotAbsence(t *testing.T) {

+ for _, tc := range []struct {

+ name, crash string

+ then string

+ }{

+ {"the sealed snapshot landed", "archive-sealed-snapshot", provision.MoveLanding},

+ {"nothing landed", "archive-journaled", provision.MoveReverting},

+ } {

+ t.Run(tc.name, func(t *testing.T) {

+ h := newCutHarness(t)

+ h.ready(t)

+ h.writeOnStop(t)

+ h.crashAt = tc.crash

+ h.cut(t)

+ h.crashPass(t)

+ first := h.journalArchive(t)

+ if err := os.RemoveAll(archiveDir(h.cfg.DataDir, p1Host, p1ID)); err != nil {

+ t.Fatal(err)

+ }

+ failed := 0

+ h.vs.deny = func(op, _, key string) error {

+ if op == "list-versions" && strings.Contains(key, moves.SnapshotsPrefix(p1Host, p1ID)) {

+ failed++

+ return errors.New("injected listing failure")

+ }

+ return nil

+ }

+ h.freshWorker()

+ h.pass(t)

+ if failed == 0 {

+ t.Fatal("the replay never listed the snapshots")

+ }

+ m := h.move(t)

+ if m.State != provision.MoveCutting || m.CutPhase != provision.CutStaged {

+ t.Fatalf("after the listing failure: %+v", m)

+ }

+ if again := h.journalArchive(t); again.Sealed != first.Sealed {

+ t.Fatalf("the listing failure changed the identity from %s to %s", first.Name, again.Name)

+ }

+ if h.seal(t).HasFiles() {

+ t.Fatal("the seal names files")

+ }

+ h.vs.deny = nil

+ h.freshWorker()

+ h.pass(t)

+ m = h.move(t)

+ if m.State != tc.then {

+ t.Fatalf("after the listing succeeded: %+v", m)

+ }

+ if tc.then == provision.MoveLanding && (m.Source.Files.Sealed != first.Sealed || !sealNamesArchive(h.seal(t), first.Sealed)) {

+ t.Fatalf("not the journalled identity: %+v", m.Source.Files)

+ }

+ })

+ }

+// Ruling 5: the archive seal's bound goes into the ledger as well as the

+// seal, and a ledger that cannot be read or written stops the phase. An

+// injected ledger-read failure after the archive is sealed leaves the

+// record staged and the ledger's bound as the stopped seal wrote it; the

+// next pass finishes and the ledger carries the archive seal's bound.

+func TestArchiveSealStopsOnALedgerFailure(t *testing.T) {

+ h := newCutHarness(t)

+ h.ready(t)

+ ledgerKey := moves.LedgerKey(p1Host, p1ID)

+ armed := false

+ h.cfg.cutFault = func(point string) error {

+ if point == "after-archive-seal" && !armed {

+ armed = true

+ h.f.failGet = func(_, key string) error {

+ if key == ledgerKey {

+ return errors.New("injected ledger read failure")

+ }

+ return nil

+ }

+ return nil

+ }

+ h.cut(t)

+ h.pass(t)

+ if !armed {

+ t.Fatal("the pass never sealed the archive")

+ }

+ m := h.move(t)

+ if m.State != provision.MoveCutting || m.CutPhase != provision.CutStaged {

+ t.Fatalf("the phase advanced over a ledger failure: %+v", m)

+ }

+ sealBound := h.seal(t).LeaseUntil

+ h.f.failGet = nil

+ // The ledger still carries the stopped seal's bound, written before the

+ // archive: never later than the archive seal's.

+ before := h.ledger(t)

+ if before.LeaseUntil.IsZero() || before.LeaseUntil.After(sealBound) {

+ t.Fatalf("ledger bound %v against the archive seal's %v", before.LeaseUntil, sealBound)

+ }

+ h.freshWorker()

+ h.pass(t)

+ if m := h.move(t); m.State != provision.MoveLanding {

+ t.Fatalf("the replay: %+v", m)

+ }

+ after := h.ledger(t)

+ if seal := h.seal(t); !after.LeaseUntil.Equal(seal.LeaseUntil) || after.LeaseUntil.Before(before.LeaseUntil) {

+ t.Fatalf("ledger bound %v, seal bound %v (was %v)", after.LeaseUntil, seal.LeaseUntil, before.LeaseUntil)

+ }

diff --git a/internal/host/movespass_test.go b/internal/host/movespass_test.go

index d6362c070..93f74b5dd 100644

--- a/internal/host/movespass_test.go

+++ b/internal/host/movespass_test.go

@@ -33,6 +33,7 @@ type fakeMoveMirror struct {

mu sync.Mutex

at []stagingCall

failAt func(op, key string) error

+ failGet func(bucket, key string) error // a plain read that fails (the ledger, movescut_test.go)

versions int

vers map[string]string

}

@@ -85,6 +86,11 @@ func (f *fakeMoveMirror) Delete(_ context.Context, bucket, key string) error {

return nil

}

func (f *fakeMoveMirror) Get(_ context.Context, bucket, key string) ([]byte, error) {

+ if f.failGet != nil {

+ if err := f.failGet(bucket, key); err != nil {

+ return nil, err

+ }

o, ok := f.buckets[bucket][key]

if !ok {

return nil, lease.ErrMirrorNotFound