amadan

carlos / platform Public

zdd-5 task 4: failure records in the agent data dir

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 505fd5c7673753321b5cf0f3774e18dd8e6edc56 parent a4dd5de
2 files changed, +368 −0
  • internal/rollout/failures.go +239 −0
  • internal/rollout/failures_test.go +129 −0
diff --git a/internal/rollout/failures.go b/internal/rollout/failures.go
new file mode 100644
index 000000000..bb119f247
--- /dev/null
+++ b/internal/rollout/failures.go
@@ -0,0 +1,239 @@
+package rollout
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "slices"
+ "sync"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// FailureFile is the failure records' home in the agent's data dir (spec
+// §8.4): on disk, so a record survives an agent restart.
+const FailureFile = "swap-failures.json"
+
+// FailureRecord holds one build off this box for instances that have not
+// served it (spec §8.4). Its scope is (Account, App, Build) on this box;
+// Host is the instance whose attempt failed, kept for status.
+type FailureRecord struct {
+ Account string `json:"account"`
+ App string `json:"app"`
+ Build string `json:"build"`
+ Host string `json:"host"`
+ Cause string `json:"cause"`
+ Mode string `json:"mode"`
+ Reason string `json:"reason"`
+ // PromotedAt is the PromotedAt of the channel pointer the build failed
+ // under (spec rev 14 §8.4). A pointer naming the same build with a newer
+ // one, which only `carlos promote --retry` or a fresh promotion can sign,
+ // clears the record: the refusal records' rule (internal/release/
+ // refusal.go), so the two kinds of record behave alike.
+ PromotedAt time.Time `json:"promoted_at"`
+ At time.Time `json:"at"`
+ // HeldDown are instances where a hold build executed and failed (spec
+ // §6.4, §8.3): nothing falls back automatically on them, so their wakes
+ // are refused until a newer build or a retry.
+ HeldDown []string `json:"held_down,omitempty"`
+ // Retried are instances `carlos instances retry` cleared: the record no
+ // longer holds them, and their next attempt is a canary.
+ Retried []string `json:"retried,omitempty"`
+}
+
+type failureFile struct {
+ Schema int `json:"schema"`
+ Records []FailureRecord `json:"records"`
+}
+
+// FailureBook is the box's failure records. An empty path keeps them in
+// memory only, which is what tests and the simulation use.
+type FailureBook struct {
+ path string
+ mu sync.Mutex
+ recs []FailureRecord
+}
+
+// OpenFailureBook loads the book from dataDir. A file that exists but cannot
+// be read is an error, never an empty book: an empty book would move every
+// held instance straight back onto the build that failed.
+func OpenFailureBook(dataDir string) (*FailureBook, error) {
+ b := &FailureBook{path: filepath.Join(dataDir, FailureFile)}
+ raw, err := os.ReadFile(b.path)
+ if errors.Is(err, os.ErrNotExist) {
+ return b, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ var f failureFile
+ if err := json.Unmarshal(raw, &f); err != nil {
+ return nil, fmt.Errorf("%s: %w", b.path, err)
+ }
+ if f.Schema != 1 {
+ return nil, fmt.Errorf("%s: unknown schema %d", b.path, f.Schema)
+ }
+ b.recs = f.Records
+ return b, nil
+}
+
+func (b *FailureBook) saveLocked() error {
+ if b.path == "" {
+ return nil
+ }
+ raw, err := json.MarshalIndent(failureFile{Schema: 1, Records: b.recs}, "", " ")
+ if err != nil {
+ return err
+ }
+ return release.WriteFileAtomic(b.path, raw, 0o644)
+}
+
+func (b *FailureBook) find(acct, app, build string) int {
+ return slices.IndexFunc(b.recs, func(r FailureRecord) bool {
+ return r.Account == acct && r.App == app && r.Build == build
+ })
+}
+
+// Record writes r unless a record for its build already stands, in which case
+// the first failure stands and only r's host is merged in: it leaves Retried
+// (its retry was spent), joins HeldDown if r holds it down, and the record
+// keeps the newer of the two PromotedAt stamps, so only a pointer signed
+// after the latest failure clears it. The in-memory record is kept even when
+// the file write fails, so the caller's transition still blocks the build;
+// the error is for the journal.
+func (b *FailureBook) Record(r FailureRecord) (bool, error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ i := b.find(r.Account, r.App, r.Build)
+ if i < 0 {
+ r.Retried = nil
+ b.recs = append(b.recs, r)
+ return true, b.saveLocked()
+ }
+ cur := &b.recs[i]
+ changed := false
+ if j := slices.Index(cur.Retried, r.Host); j >= 0 {
+ cur.Retried = slices.Delete(cur.Retried, j, j+1)
+ changed = true
+ }
+ if r.PromotedAt.After(cur.PromotedAt) {
+ cur.PromotedAt = r.PromotedAt
+ changed = true
+ }
+ for _, h := range r.HeldDown {
+ if !slices.Contains(cur.HeldDown, h) {
+ cur.HeldDown = append(cur.HeldDown, h)
+ changed = true
+ }
+ }
+ if !changed {
+ return false, nil
+ }
+ return false, b.saveLocked()
+}
+
+// Lookup returns the record for (acct, app, build), if one stands.
+func (b *FailureBook) Lookup(acct, app, build string) (FailureRecord, bool) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ if i := b.find(acct, app, build); i >= 0 {
+ return b.recs[i], true
+ }
+ return FailureRecord{}, false
+}
+
+// Blocks reports whether host may not be moved onto build (spec §8.4). An
+// instance that has served the build (served) is never blocked, and neither
+// is one a host retry cleared.
+func (b *FailureBook) Blocks(acct, app, build, host string, served bool) bool {
+ if served {
+ return false
+ }
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ i := b.find(acct, app, build)
+ return i >= 0 && !slices.Contains(b.recs[i].Retried, host)
+}
+
+// HeldDown reports whether host's wakes are refused after a failed hold
+// attempt onto build (spec §6.4).
+func (b *FailureBook) HeldDown(acct, app, build, host string) bool {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ i := b.find(acct, app, build)
+ return i >= 0 && slices.Contains(b.recs[i].HeldDown, host) && !slices.Contains(b.recs[i].Retried, host)
+}
+
+// RetryHost is `carlos instances retry <host>`: it clears the record for that
+// host only (spec §8.4). How the request reaches the box is plan 6's.
+func (b *FailureBook) RetryHost(acct, app, build, host string) (bool, error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ i := b.find(acct, app, build)
+ if i < 0 || slices.Contains(b.recs[i].Retried, host) {
+ return false, nil
+ }
+ cur := &b.recs[i]
+ cur.Retried = append(cur.Retried, host)
+ if j := slices.Index(cur.HeldDown, host); j >= 0 {
+ cur.HeldDown = slices.Delete(cur.HeldDown, j, j+1)
+ }
+ return true, b.saveLocked()
+}
+
+// ClearOnPointer is `carlos promote --retry` as this box sees it: a signed
+// channel pointer naming build whose PromotedAt is newer than the record's
+// clears the record (spec rev 14 §8.4). The pointer's signature is verified
+// by the caller's read; nobody can forge a fresh one.
+func (b *FailureBook) ClearOnPointer(acct, app, build string, promotedAt time.Time) (bool, error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ i := b.find(acct, app, build)
+ if i < 0 || !promotedAt.After(b.recs[i].PromotedAt) {
+ return false, nil
+ }
+ b.recs = slices.Delete(b.recs, i, i+1)
+ return true, b.saveLocked()
+}
+
+// ForApp returns the app's records.
+func (b *FailureBook) ForApp(acct, app string) []FailureRecord {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ var out []FailureRecord
+ for _, r := range b.recs {
+ if r.Account == acct && r.App == app {
+ out = append(out, r)
+ }
+ }
+ return out
+}
+
+// Prune drops the records for builds no route on this box still has
+// installed. "The record stays for any instance still on the failed build"
+// (spec §8.4), and only for those.
+func (b *FailureBook) Prune(installed func(acct, app, build string) bool) error {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ kept := b.recs[:0]
+ for _, r := range b.recs {
+ if installed(r.Account, r.App, r.Build) {
+ kept = append(kept, r)
+ }
+ }
+ if len(kept) == len(b.recs) {
+ return nil
+ }
+ b.recs = kept
+ return b.saveLocked()
+}
+
+// List returns a copy of every record, for status.
+func (b *FailureBook) List() []FailureRecord {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return slices.Clone(b.recs)
+}
diff --git a/internal/rollout/failures_test.go b/internal/rollout/failures_test.go
new file mode 100644
index 000000000..d70ade866
--- /dev/null
+++ b/internal/rollout/failures_test.go
@@ -0,0 +1,129 @@
+package rollout
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+var t0 = time.Unix(1_700_000_000, 0).UTC()
+
+// promotedAt is when the pointer the failures below happened under was signed.
+var promotedAt = t0.Add(-time.Hour)
+
+func rec(build, host string) FailureRecord {
+ return FailureRecord{Account: "bab", App: "blog", Build: build, Host: host, Cause: "release", Mode: "overlap",
+ Reason: "health check failed", PromotedAt: promotedAt, At: t0}
+}
+
+func TestFailureRecordBlocksOnlyInstancesThatHaveNotServedTheBuild(t *testing.T) {
+ b := &FailureBook{}
+ added, err := b.Record(rec("v2", "a.example.com"))
+ if err != nil || !added {
+ t.Fatalf("Record = %v, %v", added, err)
+ }
+ if !b.Blocks("bab", "blog", "v2", "b.example.com", false) {
+ t.Fatal("an instance that has not served v2 must be held off it")
+ }
+ if b.Blocks("bab", "blog", "v2", "c.example.com", true) {
+ t.Fatal("an instance already serving v2 must not be blocked (spec §8.4)")
+ }
+ if b.Blocks("bab", "blog", "v3", "b.example.com", false) {
+ t.Fatal("a newer installed build no longer matches the record")
+ }
+ if b.Blocks("bab", "other", "v2", "b.example.com", false) {
+ t.Fatal("the scope is (app, build, box), not the build name alone")
+ }
+ if added, _ := b.Record(rec("v2", "b.example.com")); added {
+ t.Fatal("the first failure stands; a second is not a second record")
+ }
+}
+
+// Review focus 3: the record outlives an agent restart.
+func TestFailureBookSurvivesReopen(t *testing.T) {
+ dir := t.TempDir()
+ b, err := OpenFailureBook(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ r := rec("v2", "a.example.com")
+ r.HeldDown = []string{"a.example.com"}
+ if _, err := b.Record(r); err != nil {
+ t.Fatal(err)
+ }
+ again, err := OpenFailureBook(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !again.Blocks("bab", "blog", "v2", "b.example.com", false) {
+ t.Fatal("the reopened book lost the record")
+ }
+ if !again.HeldDown("bab", "blog", "v2", "a.example.com") {
+ t.Fatal("the reopened book lost the held-down host")
+ }
+}
+
+func TestACorruptFailureFileIsAnError(t *testing.T) {
+ dir := t.TempDir()
+ if err := os.WriteFile(filepath.Join(dir, FailureFile), []byte("{nope"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := OpenFailureBook(dir); err == nil {
+ t.Fatal("a book that cannot be read must refuse, not start empty")
+ }
+}
+
+func TestRetryClearing(t *testing.T) {
+ b := &FailureBook{}
+ r := rec("v2", "a.example.com")
+ r.HeldDown = []string{"a.example.com"}
+ b.Record(r)
+
+ // carlos instances retry: one host only.
+ if ok, _ := b.RetryHost("bab", "blog", "v2", "a.example.com"); !ok {
+ t.Fatal("a host retry clears that host")
+ }
+ if b.Blocks("bab", "blog", "v2", "a.example.com", false) || b.HeldDown("bab", "blog", "v2", "a.example.com") {
+ t.Fatal("the retried host is still held")
+ }
+ if !b.Blocks("bab", "blog", "v2", "b.example.com", false) {
+ t.Fatal("a host retry cleared another host")
+ }
+ // The retried host fails the build again: it is held again.
+ b.Record(rec("v2", "a.example.com"))
+ if !b.Blocks("bab", "blog", "v2", "a.example.com", false) {
+ t.Fatal("a second failure must hold the retried host again")
+ }
+
+ // carlos promote --retry (spec rev 14 §8.4): a pointer naming the build
+ // with a NEWER PromotedAt clears the record; the pointer it failed under,
+ // seen again, does not.
+ if ok, _ := b.ClearOnPointer("bab", "blog", "v2", promotedAt); ok {
+ t.Fatal("the pointer the build failed under must not clear the record")
+ }
+ if ok, _ := b.ClearOnPointer("bab", "blog", "v2", promotedAt.Add(-time.Minute)); ok {
+ t.Fatal("an older pointer must not clear the record")
+ }
+ if ok, _ := b.ClearOnPointer("bab", "blog", "v2", promotedAt.Add(time.Second)); !ok {
+ t.Fatal("a re-signed pointer for the same build clears the record")
+ }
+ if _, ok := b.Lookup("bab", "blog", "v2"); ok {
+ t.Fatal("record still present after the retry")
+ }
+}
+
+func TestPruneKeepsRecordsSomeRouteStillInstalls(t *testing.T) {
+ b := &FailureBook{}
+ b.Record(rec("v2", "a.example.com"))
+ b.Record(rec("v3", "a.example.com"))
+ if err := b.Prune(func(_, _, build string) bool { return build == "v3" }); err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := b.Lookup("bab", "blog", "v2"); ok {
+ t.Fatal("v2 is installed nowhere and should have been pruned")
+ }
+ if _, ok := b.Lookup("bab", "blog", "v3"); !ok {
+ t.Fatal("v3 is still installed and must stay held (spec §8.4)")
+ }
+}