carlos / platform Public

settings-4 task 6: fix round 1: unreadable settings hold; a pending request relays to root; sel clears refusals

- R-SETTINGS-UNREADABLE: the box parses update settings leniently
  (ParseUpdateSettingsLenient drops Control names it does not know; the
  console keeps the strict parse). A column that still does not parse
  holds the route: nothing installed, the live link kept, and on root a
  consumed decision with reason unknown naming the parse problem.
- R-AGENT-RELAY: an agent pass that stops for root's resolution of an
  install request calls OnStageBlocked with ErrInstallRequestNeedsRoot,
  so a cold wake gets root's pass promptly.
- The up-to-date refusal clear judges sel, not the head.
- The failure-book-unreadable line prints on root's pass; the agent's
  opener logs once per distinct error. A host test pins the per-pass
  reopen and Blocks || HeldDown.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 1 week ago 9d5a4978aa8d53f042a4d4520688a54b7fe8d6ba parent f805dd7

5 files changed, +370 −12

internal/host/updatesroot.go +21 −0
internal/host/updatesroot_test.go +103 −0
internal/release/adopt.go +36 −12
internal/release/adoptsettings_test.go +180 −0
internal/release/updatesettings.go +30 −0

diff --git a/internal/host/updatesroot.go b/internal/host/updatesroot.go

index 2b41306d7..0b9c39f48 100644

--- a/internal/host/updatesroot.go

+++ b/internal/host/updatesroot.go

@@ -1,6 +1,9 @@

package host

import (

+ "log"

+ "sync"

"github.com/carlosframework/platform/internal/release"

"github.com/carlosframework/platform/internal/rollout"

)

@@ -19,9 +22,27 @@ import (

// unreadable book holds every route with settings and every managed route

// (rollout.Managed), as refuseManaged does; a default unmanaged route keeps

// today's adoption, which never read the book.

+//

+// The pass prints the unreadable-book line on root's pass only. The agent's

+// opener logs it here once per distinct error, since the fast loop runs a

+// pass every few seconds with options built once.

func updatesPassOption(dd string, root bool) release.PassOption {

+ var (

+ mu sync.Mutex

+ lastErr string

+ )

open := func() (release.FailedBuild, error) {

book, err := rollout.OpenFailureBook(dd)

+ mu.Lock()

+ said := ""

+ if err != nil {

+ said = err.Error()

+ }

+ if !root && err != nil && said != lastErr {

+ log.Printf("agent: failure book unreadable (%v): routes with update settings and managed routes install nothing", err)

+ }

+ lastErr = said

+ mu.Unlock()

if err != nil {

return nil, err

}

diff --git a/internal/host/updatesroot_test.go b/internal/host/updatesroot_test.go

new file mode 100644

index 000000000..086b6607b

--- /dev/null

+++ b/internal/host/updatesroot_test.go

@@ -0,0 +1,103 @@

+package host

+import (

+ "context"

+ "os"

+ "path/filepath"

+ "testing"

+ "github.com/carlosframework/platform/internal/registry"

+ "github.com/carlosframework/platform/internal/release"

+ "github.com/carlosframework/platform/internal/rollout"

+// Settings spec §4.3 through updatesPassOption: the option is built ONCE,

+// as fastPassOpts builds it, and still reads swap-failures.json at the start

+// of every pass. A build is failed for a host when the book blocks it or

+// holds the host down on it; a served build is exempt from Blocks only.

+func TestUpdatesPassOptionReadsTheBookEveryPass(t *testing.T) {

+ ctx := context.Background()

+ opt, data := t.TempDir(), t.TempDir()

+ store := release.NewDirStore(t.TempDir())

+ seed, pubB64, _ := release.GenerateKey()

+ priv, _ := release.ParseSigningKey(seed, "t")

+ pub, _ := release.ParsePublicKey(pubB64, "t")

+ for _, v := range []string{"v1", "v2"} {

+ m := release.Manifest{Schema: 1, App: "km", Version: v, Kind: "binary",

+ Artifacts: map[string]release.Artifact{runtimePlatform(): {}}}

+ if err := release.Ship(ctx, store, "bab", m, map[string][]byte{runtimePlatform(): []byte(v)}); err != nil {

+ t.Fatal(err)

+ }

+ if err := release.Promote(ctx, store, priv, "bab", "km", "canary/x", v); err != nil {

+ t.Fatal(err)

+ }

+ reg, err := registry.Open(filepath.Join(t.TempDir(), "r.db"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ pinned := release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "canary/x", Pin: "v1"}.Column()

+ add := func(host string, good string) {

+ t.Helper()

+ if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", App: "km", Account: "bab", Channel: "canary/x"}); err != nil {

+ t.Fatal(err)

+ }

+ if err := reg.SetUpdateSettings(host, pinned); err != nil {

+ t.Fatal(err)

+ }

+ if err := reg.SetBuildState(host, release.BuildStateCopy{Known: true}.Column()); err != nil {

+ t.Fatal(err)

+ }

+ if good != "" {

+ if err := reg.SetGoodBuild(host, registry.GoodBuild{}, registry.GoodBuild{Version: good}); err != nil {

+ t.Fatal(err)

+ }

+ opts := []release.PassOption{updatesPassOption(data, true)} // built once

+ pass := func() {

+ t.Helper()

+ if err := release.PassFilter(ctx, reg, store, pub, opt, nil, nil, opts...); err != nil {

+ t.Fatal(err)

+ }

+ live := func(host string) string {

+ v, _ := release.LiveVersion(opt, host)

+ return v

+ }

+ add("first.example.com", "")

+ pass() // no book yet: the pin installs

+ if v := live("first.example.com"); v != "v1" {

+ t.Fatalf("first live %q, want v1 with no failure records", v)

+ }

+ // The agent records v1 failed after the option was built.

+ book := `{"schema":1,"records":[{"account":"bab","app":"km","build":"v1","host":"first.example.com",

+ "cause":"health","mode":"canary","reason":"did not come up",

+ "held_down":["held.example.com"],"retried":["retried.example.com"]}]}`

+ if err := os.WriteFile(filepath.Join(data, rollout.FailureFile), []byte(book), 0o644); err != nil {

+ t.Fatal(err)

+ }

+ add("blocked.example.com", "") // Blocks: not served, not retried

+ add("served.example.com", "v1") // served: Blocks exempts, not held down

+ add("held.example.com", "v1") // served, but held down on v1: HeldDown

+ add("retried.example.com", "") // a host retry cleared it

+ pass()

+ for host, want := range map[string]string{

+ "blocked.example.com": "",

+ "served.example.com": "v1",

+ "held.example.com": "",

+ "retried.example.com": "v1",

+ } {

+ if v := live(host); v != want {

+ t.Fatalf("%s live %q, want %q", host, v, want)

+ }

+ for _, host := range []string{"blocked.example.com", "held.example.com"} {

+ c, err := release.ReadConsumed(opt, host)

+ if err != nil || c == nil || c.Reason != string(release.ReasonFailed) || c.Detail != "v1" {

+ t.Fatalf("%s consumed = %+v, %v; want reason failed naming v1", host, c, err)

+ }

diff --git a/internal/release/adopt.go b/internal/release/adopt.go

index 2a423474f..10bdc29a9 100644

--- a/internal/release/adopt.go

+++ b/internal/release/adopt.go

@@ -126,6 +126,13 @@ type passConfig struct {

// rollout imports this package. served is build == the route's GoodVersion.

type FailedBuild func(acct, app, build, host string, served bool) bool

+// ErrInstallRequestNeedsRoot is what an agent pass hands OnStageBlocked's

+// callback when it stops for an install request only root may resolve

+// (settings spec §4.3 rule 2; ruling R-AGENT-RELAY). Nothing failed: the

+// callback only asks for root's pass, and the version is the request's T as

+// this pass sees it. Never shown to members.

+var ErrInstallRequestNeedsRoot = errors.New("release: an install request waits for root's pass to resolve it")

// NagExpiredCanaries opts a Pass/PassFilter call into spec §5's nag: one

// line per expired canary/* route the call sees, written to w.

@@ -421,7 +428,7 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551

// record the agent persisted since the last one counts now.

if cfg.settings && cfg.openFailed != nil {

cfg.failed, cfg.failedErr = cfg.openFailed()

- if cfg.failedErr != nil {

+ if cfg.failedErr != nil && cfg.resolve {

fmt.Printf(" failure book unreadable (%v): routes with update settings and managed routes install nothing this pass\n", cfg.failedErr)

}

@@ -582,7 +589,7 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551

if cfg.settings {

cfg.tags = map[string]map[string]string{}

for _, rt := range routes {

- u, err := ParseUpdateSettings(rt.UpdateSettings)

+ u, err := ParseUpdateSettingsLenient(rt.UpdateSettings)

if err != nil || u.Major == nil || !adopts(rt) {

continue

}

@@ -1229,9 +1236,11 @@ func adoptHost(ctx context.Context, work hostwrite.Work, reg *registry.Registry,

// agent cannot remove root's file, so its failure is ignored and the

// next root pass retries. A record that still blocks stays: the link

// is on the refused build only when its rollback failed, and that is

- // worth showing. With no readable head nothing is known to be stale.

- if cfg.bakeDir != "" && ptr != nil {

- if r, err := ReadRefusal(cfg.bakeDir, rt.Host); err == nil && r != nil && !r.Blocks(ptr) {

+ // worth showing. The record is judged against the build this route

+ // selected (sel): under a pin or a hold, a refusal of the head is

+ // not the live build's.

+ if cfg.bakeDir != "" {

+ if r, err := ReadRefusal(cfg.bakeDir, rt.Host); err == nil && r != nil && !r.Blocks(sel) {

_ = ClearRefusal(cfg.bakeDir, rt.Host)

}

@@ -2141,13 +2150,7 @@ func settingsTarget(optDir string, bake *FirstSeen, rt registry.Route, ptr *Chan

return "", dec, true

}

acct, channel := RouteAccount(rt), routeChannel(rt)

- u, uerr := ParseUpdateSettings(rt.UpdateSettings)

- if uerr != nil {

- if cfg.resolve {

- fmt.Printf(" %s: update settings unreadable (%v): following the channel\n", rt.Host, uerr)

- }

- u = UpdateSettings{Updates: UpdatesAuto}

- }

+ u, uerr := ParseUpdateSettingsLenient(rt.UpdateSettings)

plain := uerr == nil && u.Default()

managed := cfg.managed != nil && cfg.managed(rt)

consumed, cerr := ReadConsumed(optDir, rt.Host)

@@ -2164,6 +2167,21 @@ func settingsTarget(optDir string, bake *FirstSeen, rt registry.Route, ptr *Chan

fmt.Printf(" %s: consumed file unreadable (%v): nothing installed\n", rt.Host, cerr)

return stop(TargetDecision{})

}

+ // A column that does not parse even leniently is held where it is, never

+ // read as auto (ruling R-SETTINGS-UNREADABLE): it may carry a hold this

+ // binary cannot see. Root records reason unknown, so the wake check holds

+ // too; nothing is installed and the live link stays.

+ if uerr != nil {

+ dec := TargetDecision{Reason: ReasonUnknown, Detail: uerr.Error()}

+ if cfg.resolve {

+ fmt.Printf(" %s: update settings unreadable (%v): held, nothing installed\n", rt.Host, uerr)

+ in := TargetInput{Route: rt, Pointer: ptr, Consumed: consumed, Installed: cur}

+ if err := WriteConsumed(optDir, NextConsumed(in, dec)); err != nil {

+ fmt.Printf(" %s: recording the update decision: %v\n", rt.Host, err)

+ }

+ return stop(dec)

+ }

if cfg.failedErr != nil && (!plain || managed) {

return stop(TargetDecision{}) // PassFilter said why, once

}

@@ -2196,6 +2214,12 @@ func settingsTarget(optDir string, bake *FirstSeen, rt registry.Route, ptr *Chan

// the request's: the next root pass does both. A decision that

// carries a Resolve for another reason (a pin, a hold, a failure:

// R-TARGET-FAILEDHEAD) does not depend on it, so it goes ahead.

+ // The stop asks root for a pass through the stage relay (ruling

+ // R-AGENT-RELAY), the way a stage this process cannot write does, so

+ // a cold wake waits on root's resolution rather than an hour.

+ if cfg.stageBlocked != nil && !cfg.hostWrites.Enabled {

+ cfg.stageBlocked(rt.Host, dec.Version, ErrInstallRequestNeedsRoot)

+ }

return stop(dec)

}

if dec.Version == "" {

diff --git a/internal/release/adoptsettings_test.go b/internal/release/adoptsettings_test.go

index bb2354dbd..ff7892a40 100644

--- a/internal/release/adoptsettings_test.go

+++ b/internal/release/adoptsettings_test.go

@@ -648,3 +648,183 @@ func TestNextConsumed(t *testing.T) {

})

}

+// installAt stages ver of km on this box and points host at it, as an

+// earlier pass would have.

+func installAt(t *testing.T, s Store, opt, host, ver string) {

+ t.Helper()

+ if _, err := ensureStaged(context.Background(), s, opt, "bab", "km", ver, host); err != nil {

+ t.Fatal(err)

+ }

+ if err := PointLive(opt, host, "bab", ver, "km"); err != nil {

+ t.Fatal(err)

+ }

+// R-SETTINGS-UNREADABLE (a): a control name this binary does not know (a

+// newer console's) is ignored by root; the rest of the column still holds.

+func TestAnUnknownControlNameKeepsTheHold(t *testing.T) {

+ s := NewDirStore(t.TempDir())

+ ctx := context.Background()

+ priv, pub := testKeys(t)

+ opt, reg := t.TempDir(), testRegistry(t)

+ for _, v := range []string{"v1", "v2"} {

+ shipRelease(t, s, "bab", "km", v, adoptPlatform, []byte(v))

+ if err := Promote(ctx, s, priv, "bab", "km", "canary/x", v); err != nil {

+ t.Fatal(err)

+ }

+ settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{}, BuildStateCopy{Known: true, Replica: true, State: &BuildState{LastExecuted: "v1"}})

+ if err := reg.SetUpdateSettings("a.example.com", `{"updates":"manual","channel":"canary/x","control":["schedule","pin","schedule"]}`); err != nil {

+ t.Fatal(err)

+ }

+ installAt(t, s, opt, "a.example.com", "v1")

+ passWithSettings(t, reg, s, pub, opt)

+ if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" {

+ t.Fatalf("live %q: an unknown control name lifted the manual hold", v)

+ }

+ if c, _ := ReadConsumed(opt, "a.example.com"); c == nil || c.Reason != string(ReasonManual) {

+ t.Fatalf("consumed = %+v; want reason manual", c)

+ }

+ u, err := ParseUpdateSettingsLenient(`{"updates":"manual","control":["schedule","pin","pin"]}`)

+ if err != nil || u.Updates != UpdatesManual || !reflect.DeepEqual(u.Control, []string{ControlPin}) {

+ t.Fatalf("lenient parse = %+v, %v; want manual with control [pin]", u, err)

+ }

+ if _, err := ParseUpdateSettings(`{"updates":"manual","control":["schedule"]}`); err == nil {

+ t.Fatal("the strict parse (the console's) accepted an unknown control name")

+ }

+// R-SETTINGS-UNREADABLE (b): a column that still does not parse holds the

+// route where it is, and root records reason unknown for the wake check.

+func TestAnUnreadableSettingsColumnHolds(t *testing.T) {

+ s := NewDirStore(t.TempDir())

+ ctx := context.Background()

+ priv, pub := testKeys(t)

+ opt, reg := t.TempDir(), testRegistry(t)

+ for _, v := range []string{"v1", "v2"} {

+ shipRelease(t, s, "bab", "km", v, adoptPlatform, []byte(v))

+ if err := Promote(ctx, s, priv, "bab", "km", "canary/x", v); err != nil {

+ t.Fatal(err)

+ }

+ for host, col := range map[string]string{

+ "later.example.com": `{"updates":"scheduled","channel":"canary/x"}`,

+ "garbage.example.com": `{not json`,

+ } {

+ settingsRoute(t, reg, host, "km", "canary/x", UpdateSettings{}, BuildStateCopy{Known: true, Replica: true, State: &BuildState{LastExecuted: "v1"}})

+ if err := reg.SetUpdateSettings(host, col); err != nil {

+ t.Fatal(err)

+ }

+ installAt(t, s, opt, "later.example.com", "v1")

+ var agent bytes.Buffer

+ withCapturedStdout(t, &agent, func() {

+ if err := Pass(ctx, reg, s, pub, opt, nil, WithUpdateSettings(nil, nil, false)); err != nil {

+ t.Fatal(err)

+ }

+ })

+ if strings.Contains(agent.String(), "update settings") {

+ t.Fatalf("an agent pass printed the settings line:\n%s", agent.String())

+ }

+ if c, _ := ReadConsumed(opt, "later.example.com"); c != nil {

+ t.Fatalf("an agent pass wrote %+v", c)

+ }

+ out := passWithSettings(t, reg, s, pub, opt)

+ for host, want := range map[string]string{"later.example.com": "v1", "garbage.example.com": ""} {

+ if v, _ := LiveVersion(opt, host); v != want {

+ t.Fatalf("%s live %q, want %q kept", host, v, want)

+ }

+ c, err := ReadConsumed(opt, host)

+ if err != nil || c == nil || c.Reason != string(ReasonUnknown) || !strings.Contains(c.Detail, "update settings") || c.Desired != "" {

+ t.Fatalf("%s consumed = %+v, %v; want reason unknown naming the parse problem", host, c, err)

+ }

+ if !strings.Contains(out, "update settings unreadable") {

+ t.Fatalf("root printed no line:\n%s", out)

+ }

+// R-AGENT-RELAY: an agent pass that stops for root's resolution asks root

+// for a pass, once, through the stage relay, with the request's sentinel.

+func TestAnAgentPassRelaysAPendingRequestToRoot(t *testing.T) {

+ s := NewDirStore(t.TempDir())

+ ctx := context.Background()

+ priv, pub := testKeys(t)

+ opt, reg := t.TempDir(), testRegistry(t)

+ shipRelease(t, s, "bab", "km", "v1", adoptPlatform, []byte("v1"))

+ if err := Promote(ctx, s, priv, "bab", "km", "canary/x", "v1"); err != nil {

+ t.Fatal(err)

+ }

+ settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesManual, Channel: "canary/x", InstallRequest: 1},

+ BuildStateCopy{Known: true, Replica: true})

+ type relay struct {

+ host, ver string

+ err error

+ }

+ var got []relay

+ var out bytes.Buffer

+ withCapturedStdout(t, &out, func() {

+ err := MaterializeHost(ctx, reg, s, pub, opt, t.TempDir(), "a.example.com",

+ OnStageBlocked(func(h, v string, err error) { got = append(got, relay{h, v, err}) }),

+ WithUpdateSettings(nil, nil, false))

+ if err == nil {

+ t.Fatal("materialized a request root has not resolved")

+ }

+ })

+ if len(got) != 1 || got[0].host != "a.example.com" || got[0].ver != "v1" || !errors.Is(got[0].err, ErrInstallRequestNeedsRoot) {

+ t.Fatalf("relays = %+v; want exactly one for a.example.com v1 with ErrInstallRequestNeedsRoot", got)

+ }

+ // Root's pass resolves; the relay's head is then installed.

+ passWithSettings(t, reg, s, pub, opt)

+ if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" {

+ t.Fatalf("live %q after root's pass", v)

+ }

+// R-T6-MISC fix round 1: the up-to-date refusal clear judges the selected

+// build. Pinned and live on v1: a refusal of the head v2 is stale for this

+// route and cleared; a refusal of v1 itself (its rollback failed) stays.

+func TestTheLiveBuildsRefusalIsJudgedOnTheSelectedBuild(t *testing.T) {

+ s := NewDirStore(t.TempDir())

+ ctx := context.Background()

+ priv, pub := testKeys(t)

+ opt, reg, bake := t.TempDir(), testRegistry(t), t.TempDir()

+ for _, v := range []string{"v1", "v2"} {

+ shipRelease(t, s, "bab", "km", v, adoptPlatform, []byte(v))

+ if err := Promote(ctx, s, priv, "bab", "km", "canary/x", v); err != nil {

+ t.Fatal(err)

+ }

+ head, err := Channel(ctx, s, pub, "bab", "km", "canary/x")

+ if err != nil {

+ t.Fatal(err)

+ }

+ settingsRoute(t, reg, "a.example.com", "km", "canary/x", UpdateSettings{Updates: UpdatesAuto, Channel: "canary/x", Pin: "v1"}, BuildStateCopy{Known: true})

+ installAt(t, s, opt, "a.example.com", "v1")

+ pass := func() {

+ var out bytes.Buffer

+ withCapturedStdout(t, &out, func() {

+ if err := Pass(ctx, reg, s, pub, opt, nil, BakeDir(bake), WithUpdateSettings(nil, nil, true)); err != nil {

+ t.Fatal(err)

+ }

+ })

+ }

+ if err := WriteRefusal(bake, Refusal{Host: "a.example.com", App: "km", Version: "v2", PointerPromotedAt: head.PromotedAt, Reason: "did not come up"}); err != nil {

+ t.Fatal(err)

+ }

+ pass()

+ if r, err := ReadRefusal(bake, "a.example.com"); err != nil || r != nil {

+ t.Fatalf("refusal = %+v, %v; the head's refusal was kept as though it were the live build's", r, err)

+ }

+ if err := WriteRefusal(bake, Refusal{Host: "a.example.com", App: "km", Version: "v1", Reason: "did not come up"}); err != nil {

+ t.Fatal(err)

+ }

+ pass()

+ if r, err := ReadRefusal(bake, "a.example.com"); err != nil || r == nil || r.Version != "v1" {

+ t.Fatalf("refusal = %+v, %v; the live build's own refusal was cleared", r, err)

+ }

+ if v, _ := LiveVersion(opt, "a.example.com"); v != "v1" {

+ t.Fatalf("live %q", v)

+ }

diff --git a/internal/release/updatesettings.go b/internal/release/updatesettings.go

index e2faaab6d..465cf5236 100644

--- a/internal/release/updatesettings.go

+++ b/internal/release/updatesettings.go

@@ -87,3 +87,33 @@ func ValidControl(names []string) bool {

}

return true

}

+// ParseUpdateSettingsLenient is the box's read of the column (root's pass,

+// the agent's passes and the held wake check): ParseUpdateSettings with

+// Control names this binary does not know, and repeats, dropped first.

+// Control is the console's business (who may change what); the box never

+// acts on it, so a name a newer console or policy added must not make the

+// whole column unreadable and drop the hold it carries (ruling

+// R-SETTINGS-UNREADABLE). Every other field is validated as strictly as

+// ParseUpdateSettings does; the console keeps the strict parse.

+func ParseUpdateSettingsLenient(col string) (UpdateSettings, error) {

+ if col == "" {

+ return ParseUpdateSettings(col)

+ }

+ var u UpdateSettings

+ if err := json.Unmarshal([]byte(col), &u); err != nil {

+ return UpdateSettings{}, fmt.Errorf("update settings: %v", err)

+ }

+ var known []string

+ for _, n := range u.Control {

+ if ValidControl([]string{n}) && !slices.Contains(known, n) {

+ known = append(known, n)

+ }

+ u.Control = known

+ b, err := json.Marshal(u)

+ if err != nil {

+ return UpdateSettings{}, fmt.Errorf("update settings: %v", err)

+ }

+ return ParseUpdateSettings(string(b))