| 1 | //go:build unix |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "os" |
| 10 | "path/filepath" |
| 11 | "slices" |
| 12 | "strings" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/activator" |
| 17 | "github.com/carlosframework/platform/internal/hostbox" |
| 18 | "github.com/carlosframework/platform/internal/hostclaim" |
| 19 | "github.com/carlosframework/platform/internal/hostrelocate" |
| 20 | "github.com/carlosframework/platform/internal/registry" |
| 21 | "github.com/carlosframework/platform/internal/release" |
| 22 | ) |
| 23 | |
| 24 | // Ruling 66 (Astra round 3 finding 2) and ruling 76 (Astra round 6 |
| 25 | // findings 1 and 3): after the serving CAS, the target discharges main's |
| 26 | // participation, releases its row's relocating hold, lifts the fence its |
| 27 | // probe stop installed under the persisted owner, and marks the completed |
| 28 | // relocation discharged, in that order. A refused discharge leaves all of |
| 29 | // it, so the edge keeps the row held, the activator wakes nothing and no |
| 30 | // successor starts; a holder moved off the probe session while the row is |
| 31 | // held is refused; another operation's fence is never lifted; a replay |
| 32 | // after the hold went discharges again (positive evidence, never the |
| 33 | // released hold's inference), then lifts and marks; a row held by another |
| 34 | // op, or a record whose stop was on another box, is refused with nothing |
| 35 | // done; a claim that does not serve on this box is left alone and is |
| 36 | // ErrChanged. |
| 37 | func TestRelocationReleaseHoldWaitsForDischarge(t *testing.T) { |
| 38 | t.Parallel() |
| 39 | const host, box = "shop.example", "dst" |
| 40 | ctx := t.Context() |
| 41 | created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| 42 | inc, err := hostclaim.Incarnation("acct", "app", host, created) |
| 43 | if err != nil { |
| 44 | t.Fatal(err) |
| 45 | } |
| 46 | done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: box, Session: 12, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}) |
| 47 | claim := func(change func(*hostclaim.Claim)) *hostclaim.Claim { |
| 48 | c := &hostclaim.Claim{V: 2, Gen: 31, Host: host, |
| 49 | Holder: &hostclaim.Holder{Family: "instance", Key: "apps/acct/app/instances/" + host + ".json", Account: "acct", App: "app", Incarnation: inc, CreatedAt: created, |
| 50 | Location: "instances/" + host, Stamp: 9, Session: 12, Generation: "P1", WriterBox: box, Admissions: []hostclaim.AdmissionRecord{{Session: 12, Box: box, Generation: "P1", Mode: "cold", Region: "ie"}}}, |
| 51 | Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: box, Participants: []string{box}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}}} |
| 52 | if change != nil { |
| 53 | change(c) |
| 54 | } |
| 55 | return c |
| 56 | } |
| 57 | type world struct { |
| 58 | order []string |
| 59 | store release.Store |
| 60 | reg *registry.Registry |
| 61 | act *stopProbeActivator |
| 62 | owner string |
| 63 | marker string |
| 64 | discharge error |
| 65 | // heldAt is, per discharge, whether the row was still held for |
| 66 | // rel-1 when it ran. |
| 67 | heldAt []bool |
| 68 | hold func(context.Context, string, string) error |
| 69 | } |
| 70 | setup := func(t *testing.T, c *hostclaim.Claim) *world { |
| 71 | t.Helper() |
| 72 | w := &world{store: release.NewDirStore(t.TempDir())} |
| 73 | raw, _ := json.Marshal(c) |
| 74 | key, _ := hostclaim.Key(host) |
| 75 | if _, err := w.store.Put(ctx, key, raw, ""); err != nil { |
| 76 | t.Fatal(err) |
| 77 | } |
| 78 | if w.reg, err = registry.Open(filepath.Join(t.TempDir(), "registry.db")); err != nil { |
| 79 | t.Fatal(err) |
| 80 | } |
| 81 | t.Cleanup(func() { w.reg.Close() }) |
| 82 | if err := w.reg.Upsert(registry.Route{Host: host, Status: "relocating", HoldOp: "rel-1", Socket: "/run/shop.sock"}); err != nil { |
| 83 | t.Fatal(err) |
| 84 | } |
| 85 | // The probe stop's fence, under the owner it persisted. |
| 86 | dataDir := t.TempDir() |
| 87 | marker, err := relocationProbeFencePath(dataDir, "rel-1") |
| 88 | if err != nil { |
| 89 | t.Fatal(err) |
| 90 | } |
| 91 | w.marker = marker |
| 92 | if w.owner, err = relocationProbeFenceOwner(marker, host, 12); err != nil { |
| 93 | t.Fatal(err) |
| 94 | } |
| 95 | w.act = newStopProbeActivator(&w.order) |
| 96 | fctx, cancel := context.WithTimeout(ctx, time.Second) |
| 97 | defer cancel() |
| 98 | if _, err := w.act.svc.FenceAndStopAs(fctx, host, 12, w.owner); err != nil && !errors.Is(err, activator.ErrStopUnproven) { |
| 99 | t.Fatal(err) |
| 100 | } |
| 101 | // The claim writes go through markStore, so the mark has its place |
| 102 | // in the order. |
| 103 | a := ClaimAssembly{Claims: markStore{Store: w.store, order: &w.order}, stopLocal: func(context.Context) (claimStopLocal, error) { |
| 104 | return claimStopLocal{Actor: hostbox.Actor{Box: box}}, nil |
| 105 | }} |
| 106 | cfg := RelocationAssemblyConfig{Registry: w.reg, Activator: w.act, DataDir: dataDir, Discharge: func(_ context.Context, o hostclaim.Observed) error { |
| 107 | row, _ := w.reg.Get(host) |
| 108 | w.heldAt = append(w.heldAt, row != nil && row.HoldOp == "rel-1") |
| 109 | w.order = append(w.order, "discharge") |
| 110 | return w.discharge |
| 111 | }} |
| 112 | w.hold = a.relocationReleaseHold(cfg) |
| 113 | return w |
| 114 | } |
| 115 | held := func(w *world) bool { |
| 116 | row, err := w.reg.Get(host) |
| 117 | return err == nil && row != nil && row.Status == "relocating" && row.HoldOp == "rel-1" |
| 118 | } |
| 119 | marked := func(w *world) bool { |
| 120 | obs, err := hostclaim.Read(ctx, w.store, host) |
| 121 | if err != nil { |
| 122 | t.Fatal(err) |
| 123 | } |
| 124 | r, ok, err := hostclaim.ReadCompletedRelocation(obs.Claim) |
| 125 | if err != nil { |
| 126 | t.Fatal(err) |
| 127 | } |
| 128 | return ok && r.Discharged |
| 129 | } |
| 130 | t.Run("the discharge first, then the hold, the fence and the mark", func(t *testing.T) { |
| 131 | w := setup(t, claim(nil)) |
| 132 | if err := w.hold(ctx, host, "rel-1"); err != nil { |
| 133 | t.Fatal(err) |
| 134 | } |
| 135 | if want := []string{"discharge", "lift", "mark"}; !slices.Equal(w.order, want) || !slices.Equal(w.heldAt, []bool{true}) || held(w) || w.act.svc.Fenced(host) || !marked(w) { |
| 136 | t.Fatalf("order %v held at discharge %v held=%v fenced=%v marked=%v", w.order, w.heldAt, held(w), w.act.svc.Fenced(host), marked(w)) |
| 137 | } |
| 138 | // A replay of a marked relocation does nothing at all. |
| 139 | w.order = nil |
| 140 | if err := w.hold(ctx, host, "rel-1"); err != nil || len(w.order) != 0 { |
| 141 | t.Fatalf("replay: %v %v", err, w.order) |
| 142 | } |
| 143 | }) |
| 144 | t.Run("a refused discharge keeps everything", func(t *testing.T) { |
| 145 | w := setup(t, claim(nil)) |
| 146 | w.discharge = errors.New("the participation slot is still open") |
| 147 | if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, w.discharge) { |
| 148 | t.Fatalf("got %v", err) |
| 149 | } |
| 150 | if want := []string{"discharge"}; !slices.Equal(w.order, want) || !held(w) || w.act.svc.FenceGeneration(host) != 12 || marked(w) { |
| 151 | t.Fatalf("order %v held=%v fence=%d marked=%v", w.order, held(w), w.act.svc.FenceGeneration(host), marked(w)) |
| 152 | } |
| 153 | }) |
| 154 | t.Run("a row held by another op is refused with nothing done", func(t *testing.T) { |
| 155 | w := setup(t, claim(nil)) |
| 156 | if ok, err := w.reg.HandHold(host, "relocating", "rel-9", "rel-1"); err != nil || !ok { |
| 157 | t.Fatalf("hand the hold to rel-9: %v %v", ok, err) |
| 158 | } |
| 159 | err := w.hold(ctx, host, "rel-1") |
| 160 | row, _ := w.reg.Get(host) |
| 161 | if !errors.Is(err, hostrelocate.ErrRefused) || len(w.order) != 0 || row == nil || row.HoldOp != "rel-9" || w.act.svc.FenceGeneration(host) != 12 || marked(w) { |
| 162 | t.Fatalf("%v order %v row %+v fence=%d marked=%v", err, w.order, row, w.act.svc.FenceGeneration(host), marked(w)) |
| 163 | } |
| 164 | }) |
| 165 | t.Run("a record whose stop was on another box is refused with nothing done", func(t *testing.T) { |
| 166 | other, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: "src", Session: 12, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}) |
| 167 | w := setup(t, claim(func(c *hostclaim.Claim) { c.Op.Journal[hostclaim.JournalCompleted] = string(other) })) |
| 168 | if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, hostrelocate.ErrRefused) || len(w.order) != 0 || !held(w) || w.act.svc.FenceGeneration(host) != 12 || marked(w) { |
| 169 | t.Fatalf("%v order %v held=%v fence=%d marked=%v", err, w.order, held(w), w.act.svc.FenceGeneration(host), marked(w)) |
| 170 | } |
| 171 | }) |
| 172 | t.Run("another operation's fence after a restart stays", func(t *testing.T) { |
| 173 | w := setup(t, claim(nil)) |
| 174 | w.act.svc = &activator.Service{} // the restart: this box's fence is gone |
| 175 | w.act.foreignFence(t, host, 12) |
| 176 | if err := w.hold(ctx, host, "rel-1"); err != nil { |
| 177 | t.Fatal(err) |
| 178 | } |
| 179 | if held(w) || w.act.svc.FenceGeneration(host) != 12 || !marked(w) { |
| 180 | t.Fatalf("held=%v fence=%d marked=%v", held(w), w.act.svc.FenceGeneration(host), marked(w)) |
| 181 | } |
| 182 | }) |
| 183 | t.Run("a replay after the hold went discharges, lifts and marks", func(t *testing.T) { |
| 184 | // The hold was released, then the agent stopped before the lift; |
| 185 | // a serving wake has since moved the holder on. The discharge runs |
| 186 | // again, without the holder check, and its answer is the evidence. |
| 187 | w := setup(t, claim(func(c *hostclaim.Claim) { c.Holder.Session = 13 })) |
| 188 | if _, err := w.reg.ReleaseHold(host, "rel-1"); err != nil { |
| 189 | t.Fatal(err) |
| 190 | } |
| 191 | if err := w.hold(ctx, host, "rel-1"); err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | if want := []string{"discharge", "lift", "mark"}; !slices.Equal(w.order, want) || !slices.Equal(w.heldAt, []bool{false}) || w.act.svc.Fenced(host) || !marked(w) { |
| 195 | t.Fatalf("order %v held at discharge %v fenced=%v marked=%v", w.order, w.heldAt, w.act.svc.Fenced(host), marked(w)) |
| 196 | } |
| 197 | }) |
| 198 | t.Run("a replay after the hold went with a refused discharge keeps the fence and the record", func(t *testing.T) { |
| 199 | w := setup(t, claim(nil)) |
| 200 | if _, err := w.reg.ReleaseHold(host, "rel-1"); err != nil { |
| 201 | t.Fatal(err) |
| 202 | } |
| 203 | w.discharge = errors.New("the participation slot is still open") |
| 204 | if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, w.discharge) { |
| 205 | t.Fatalf("got %v", err) |
| 206 | } |
| 207 | if want := []string{"discharge"}; !slices.Equal(w.order, want) || w.act.svc.FenceGeneration(host) != 12 || marked(w) { |
| 208 | t.Fatalf("order %v fence=%d marked=%v", w.order, w.act.svc.FenceGeneration(host), marked(w)) |
| 209 | } |
| 210 | }) |
| 211 | t.Run("a holder moved off the probe session while held", func(t *testing.T) { |
| 212 | w := setup(t, claim(func(c *hostclaim.Claim) { c.Holder.Session, c.Holder.Stamp = 40, 40 })) |
| 213 | if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, hostrelocate.ErrRefused) || len(w.order) != 0 || !held(w) || marked(w) { |
| 214 | t.Fatalf("%v order %v held=%v", err, w.order, held(w)) |
| 215 | } |
| 216 | }) |
| 217 | t.Run("no row is refused", func(t *testing.T) { |
| 218 | w := setup(t, claim(nil)) |
| 219 | if err := w.reg.Delete(host); err != nil { |
| 220 | t.Fatal(err) |
| 221 | } |
| 222 | if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, hostrelocate.ErrRefused) || len(w.order) != 0 || marked(w) { |
| 223 | t.Fatalf("%v order %v", err, w.order) |
| 224 | } |
| 225 | }) |
| 226 | for name, change := range map[string]func(*hostclaim.Claim){ |
| 227 | "still relocating": func(c *hostclaim.Claim) { |
| 228 | c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Exec: box, Participants: []string{"src", box}, |
| 229 | Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": box}} |
| 230 | }, |
| 231 | "serving on another box": func(c *hostclaim.Claim) { |
| 232 | c.Op.Exec, c.Op.Participants, c.Holder.WriterBox = "src", []string{"src"}, "src" |
| 233 | }, |
| 234 | } { |
| 235 | t.Run(name, func(t *testing.T) { |
| 236 | w := setup(t, claim(change)) |
| 237 | if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, hostbox.ErrChanged) || len(w.order) != 0 || !held(w) || w.act.svc.FenceGeneration(host) != 12 { |
| 238 | t.Fatalf("%v order %v held=%v", err, w.order, held(w)) |
| 239 | } |
| 240 | }) |
| 241 | } |
| 242 | t.Run("an absent claim", func(t *testing.T) { |
| 243 | w := setup(t, claim(nil)) |
| 244 | key, _ := hostclaim.Key(host) |
| 245 | if err := w.store.Delete(ctx, key); err != nil { |
| 246 | t.Fatal(err) |
| 247 | } |
| 248 | if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, hostbox.ErrChanged) || len(w.order) != 0 || !held(w) || w.act.svc.FenceGeneration(host) != 12 { |
| 249 | t.Fatalf("%v order %v held=%v", err, w.order, held(w)) |
| 250 | } |
| 251 | }) |
| 252 | // Absence is never evidence (the Global Constraint): the hold release |
| 253 | // reads its evidence before it releases anything, and without it |
| 254 | // refuses with nothing done: the row stays held, the probe's fence |
| 255 | // stands and the record is not marked. |
| 256 | t.Run("an unreadable completion record", func(t *testing.T) { |
| 257 | w := setup(t, claim(func(c *hostclaim.Claim) { c.Op.Journal[hostclaim.JournalCompleted] = "{not json" })) |
| 258 | if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, hostclaim.ErrRelocation) || len(w.order) != 0 || !held(w) || w.act.svc.FenceGeneration(host) != 12 { |
| 259 | t.Fatalf("%v order %v held=%v fence=%d", err, w.order, held(w), w.act.svc.FenceGeneration(host)) |
| 260 | } |
| 261 | }) |
| 262 | for name, change := range map[string]func(*testing.T, string){ |
| 263 | "no probe-fence owner": func(t *testing.T, path string) { |
| 264 | if err := os.Remove(path); err != nil { |
| 265 | t.Fatal(err) |
| 266 | } |
| 267 | }, |
| 268 | "an unreadable probe-fence owner": func(t *testing.T, path string) { |
| 269 | if err := os.WriteFile(path, []byte("{not json"), 0o600); err != nil { |
| 270 | t.Fatal(err) |
| 271 | } |
| 272 | }, |
| 273 | "a probe-fence owner for another session": func(t *testing.T, path string) { |
| 274 | if err := os.WriteFile(path, []byte(`{"host":"shop.example","session":13,"owner":"`+strings.Repeat("b", 32)+`"}`), 0o600); err != nil { |
| 275 | t.Fatal(err) |
| 276 | } |
| 277 | }, |
| 278 | } { |
| 279 | t.Run(name, func(t *testing.T) { |
| 280 | w := setup(t, claim(nil)) |
| 281 | change(t, w.marker) |
| 282 | err := w.hold(ctx, host, "rel-1") |
| 283 | if !errors.Is(err, hostrelocate.ErrRefused) || len(w.order) != 0 || !held(w) || w.act.svc.FenceGeneration(host) != 12 || marked(w) { |
| 284 | t.Fatalf("%v order %v held=%v fence=%d marked=%v", err, w.order, held(w), w.act.svc.FenceGeneration(host), marked(w)) |
| 285 | } |
| 286 | // An unreadable owner is reported as such, never as a missing |
| 287 | // one: the operator who reads the refusal must find the file. |
| 288 | if unreadable := strings.Contains(name, "unreadable"); unreadable != strings.Contains(err.Error(), "cannot be read") { |
| 289 | t.Fatalf("the refusal names the wrong reason: %v", err) |
| 290 | } |
| 291 | }) |
| 292 | } |
| 293 | } |
| 294 | |
| 295 | // Astra round 6, finding 3, through the target's real hold release and |
| 296 | // the real start CAS: a successor relocation from the box a relocation |
| 297 | // completed on starts only once that box has discharged its probe's |
| 298 | // participation and marked it. |
| 299 | func TestRelocationSuccessorStartsOnlyAfterTheTargetsDischarge(t *testing.T) { |
| 300 | const host, box = "shop.example", "dst" |
| 301 | ctx := t.Context() |
| 302 | created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| 303 | inc, err := hostclaim.Incarnation("acct", "app", host, created) |
| 304 | if err != nil { |
| 305 | t.Fatal(err) |
| 306 | } |
| 307 | done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: box, Session: 12, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}) |
| 308 | c := &hostclaim.Claim{V: 2, Gen: 31, Host: host, |
| 309 | Holder: &hostclaim.Holder{Family: "instance", Key: "apps/acct/app/instances/" + host + ".json", Account: "acct", App: "app", Incarnation: inc, CreatedAt: created, |
| 310 | Location: "instances/" + host, Stamp: 9, Session: 12, Generation: "P1", WriterBox: box, Admissions: []hostclaim.AdmissionRecord{{Session: 12, Box: box, Generation: "P1", Mode: "cold", Region: "ie"}}}, |
| 311 | Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: box, Participants: []string{box}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}}} |
| 312 | store := release.NewDirStore(t.TempDir()) |
| 313 | raw, _ := json.Marshal(c) |
| 314 | key, _ := hostclaim.Key(host) |
| 315 | if _, err := store.Put(ctx, key, raw, ""); err != nil { |
| 316 | t.Fatal(err) |
| 317 | } |
| 318 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 319 | if err != nil { |
| 320 | t.Fatal(err) |
| 321 | } |
| 322 | defer reg.Close() |
| 323 | if err := reg.Upsert(registry.Route{Host: host, Status: "relocating", HoldOp: "rel-1", Socket: "/run/shop.sock"}); err != nil { |
| 324 | t.Fatal(err) |
| 325 | } |
| 326 | var order []string |
| 327 | refused := errors.New("the participation slot is still open") |
| 328 | discharge := refused |
| 329 | a := ClaimAssembly{Claims: store, stopLocal: func(context.Context) (claimStopLocal, error) { |
| 330 | return claimStopLocal{Actor: hostbox.Actor{Box: box}}, nil |
| 331 | }} |
| 332 | // The probe stop's persisted owner: the hold release reads it before |
| 333 | // it releases anything. |
| 334 | dataDir := t.TempDir() |
| 335 | marker, err := relocationProbeFencePath(dataDir, "rel-1") |
| 336 | if err != nil { |
| 337 | t.Fatal(err) |
| 338 | } |
| 339 | if _, err := relocationProbeFenceOwner(marker, host, 12); err != nil { |
| 340 | t.Fatal(err) |
| 341 | } |
| 342 | hold := a.relocationReleaseHold(RelocationAssemblyConfig{Registry: reg, Activator: newStopProbeActivator(&order), DataDir: dataDir, |
| 343 | Discharge: func(context.Context, hostclaim.Observed) error { return discharge }}) |
| 344 | start := func() error { |
| 345 | obs, err := hostclaim.Read(ctx, store, host) |
| 346 | if err != nil { |
| 347 | t.Fatal(err) |
| 348 | } |
| 349 | _, err = hostclaim.StartRelocation(ctx, store, obs, hostclaim.RelocationStart{OpID: "rel-2", By: "box:wrk", Now: created.Add(3 * time.Hour), Region: "ie", Reason: hostclaim.ReasonDrain, |
| 350 | SourceBox: box, SourceEnrollment: strings.Repeat("c", 32), TargetBox: "spa", TargetEnrollment: strings.Repeat("d", 32), DirectoryRevision: 2, AdmissionID: "adm-2"}, nil) |
| 351 | return err |
| 352 | } |
| 353 | if err := hold(ctx, host, "rel-1"); !errors.Is(err, refused) { |
| 354 | t.Fatalf("hold: %v", err) |
| 355 | } |
| 356 | if err := start(); !errors.Is(err, hostclaim.ErrPendingChore) { |
| 357 | t.Fatalf("the successor started before the target's discharge: %v", err) |
| 358 | } |
| 359 | discharge = nil |
| 360 | if err := hold(ctx, host, "rel-1"); err != nil { |
| 361 | t.Fatal(err) |
| 362 | } |
| 363 | if err := start(); err != nil { |
| 364 | t.Fatalf("the successor after the discharge: %v", err) |
| 365 | } |
| 366 | } |
| 367 | |
| 368 | // markStore records every claim write in order as "mark": the hold |
| 369 | // release writes the claim once, to mark the completion discharged. |
| 370 | type markStore struct { |
| 371 | release.Store |
| 372 | order *[]string |
| 373 | } |
| 374 | |
| 375 | func (s markStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| 376 | *s.order = append(*s.order, "mark") |
| 377 | return s.Store.Put(ctx, key, body, ifMatch) |
| 378 | } |
| 379 | |
amadan