carlos / platform Public

console: transient refusals are typed where they are built, not recognised by their words

offlineRefusal gains transient, set on the refusals that clear on their
own (a report not current, an unread fleet, roster, manifest, claim,
identity record or reservation table, the source's facts not yet
measured). The doors answer every refusal as before; the self-serve job
waits out exactly the transient ones. TestSelfServeTransientRefusalsAreTyped.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

claude 1 day ago f2b4b38f3ab2927220eb44049bb77aef395c9c26 parent 703cc95

5 files changed, +47 −23

internal/console/apiinstancemove_offline.go +16 −12
internal/console/offlinereserve.go +1 −1
internal/console/selfserve_checks_test.go +25 −0
internal/console/selfserve_door.go +3 −3
internal/console/selfserve_job.go +2 −7

diff --git a/internal/console/apiinstancemove_offline.go b/internal/console/apiinstancemove_offline.go

index a9b8b5e64..c32c38773 100644

--- a/internal/console/apiinstancemove_offline.go

+++ b/internal/console/apiinstancemove_offline.go

@@ -137,13 +137,17 @@ func (s *Server) offlineWaitRefusal(box, from, host string) *offlineRefusal {

// (harness Run 9, finding 7: flip answered MV-O31 for a stale report, and an

// operator went looking for a process that was not there).

func offlineStaleRefusal(box string) *offlineRefusal {

- return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("the report from %s is not current. it reports again within a minute. try again then", box)}

+ return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("the report from %s is not current. it reports again within a minute. try again then", box), transient: true}

}

// offlineRefusal is what a door refuses with: the status and the sentence.

+// transient marks a refusal that clears on its own (a read that failed, a

+// report not current yet): the doors answer it as any other, and the

+// self-serve job waits it out instead of aborting the move (Task 7).

type offlineRefusal struct {

- status int

- msg string

+ status int

+ msg string

+ transient bool

}

func writeRefusal(w http.ResponseWriter, ref *offlineRefusal) {

@@ -454,7 +458,7 @@ func (s *Server) offlineSelect(ctx context.Context, f offlineFleet, host, from,

// claim backfill (MV-O20). A read that fails is a 502 and never an absence.

func (s *Server) offlineClaimEvidence(ctx context.Context, acct, app, host string, rec *provision.Record) *offlineRefusal {

has := &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s has a host claim. offline moves are only for hosts without one", host)}

- unreadable := &offlineRefusal{status: http.StatusBadGateway, msg: "could not read the host claim"}

+ unreadable := &offlineRefusal{status: http.StatusBadGateway, msg: "could not read the host claim", transient: true}

if rec.Acquisition != "" {

return has

}

@@ -543,14 +547,14 @@ func (s *Server) offlineRegionChecks(ctx context.Context, acct, app, host string

m, err := provision.GetManifest(ctx, s.cfg.Releases, acct, app)

if err != nil && !errors.Is(err, os.ErrNotExist) {

log.Printf("console: offline move %s: manifest: %v", host, err)

- return offlineSelection{}, &offlineRefusal{status: http.StatusServiceUnavailable, msg: "could not read the app's instances manifest. try again"}

+ return offlineSelection{}, &offlineRefusal{status: http.StatusServiceUnavailable, msg: "could not read the app's instances manifest. try again", transient: true}

}

if m != nil && !s.moveRegionAllowed(m.Regions, to) {

return offlineSelection{}, &offlineRefusal{status: http.StatusBadRequest, msg: "region " + region + " is not one of this app's regions: " + strings.Join(m.Regions, ", ")}

}

fleet, ok := s.readOfflineFleet(ctx)

if !ok {

- return offlineSelection{}, &offlineRefusal{status: http.StatusConflict, msg: "the fleet's edges files could not all be read. try again"}

+ return offlineSelection{}, &offlineRefusal{status: http.StatusConflict, msg: "the fleet's edges files could not all be read. try again", transient: true}

}

if to == "" || rec.Region == "" {

// A box that sets CARLOS_REGION to the home's name is home to this

@@ -701,7 +705,7 @@ func (s *Server) participantIdentityFor(ctx context.Context, f offlineFleet) (pa

case errors.Is(err, os.ErrNotExist):

case err != nil:

log.Printf("console: self-serve move: the host roster: %v", err)

- return participantIdentity{}, &offlineRefusal{status: http.StatusServiceUnavailable, msg: "could not read the host roster. try again"}

+ return participantIdentity{}, &offlineRefusal{status: http.StatusServiceUnavailable, msg: "could not read the host roster. try again", transient: true}

default:

refs := make([]string, 0, len(o.Roster.Boxes))

for ref := range o.Roster.Boxes {

@@ -749,7 +753,7 @@ func (s *Server) pinParticipant(ctx context.Context, f offlineFleet, b edgesFile

return p, nil

}

if !current {

- return provision.Participant{}, &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("the box %s has not reported recently", b.Box)}

+ return provision.Participant{}, &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("the box %s has not reported recently", b.Box), transient: true}

}

raw, err := boxid.Load(ctx, s.cfg.Releases, p.BoxID)

switch {

@@ -759,7 +763,7 @@ func (s *Server) pinParticipant(ctx context.Context, f offlineFleet, b edgesFile

return provision.Participant{}, &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("box %s has an identity record that does not verify", b.Box)}

case err != nil:

log.Printf("console: self-serve move: the identity record of %s (%s): %v", b.Box, p.BoxID, err)

- return provision.Participant{}, &offlineRefusal{status: http.StatusBadGateway, msg: fmt.Sprintf("could not read the identity record for box %s", b.Box)}

+ return provision.Participant{}, &offlineRefusal{status: http.StatusBadGateway, msg: fmt.Sprintf("could not read the identity record for box %s", b.Box), transient: true}

}

ev, err := id.verifier.Enroll(raw, boxid.Expect{Box: p.BoxID, AWSRegion: id.aws[code], Roster: id.roster})

if err != nil || ev.Cloud.InstanceID != p.InstanceID {

@@ -1177,7 +1181,7 @@ func (s *Server) offlineStepFor(ctx context.Context, d offlineDoor, mv *provisio

// door's, so the view's answer stays main's.

fleet, ok := s.readOfflineFleet(ctx)

if !ok {

- return offlineStep{}, &offlineRefusal{status: http.StatusConflict, msg: "the fleet's edges files could not all be read. try again"}

+ return offlineStep{}, &offlineRefusal{status: http.StatusConflict, msg: "the fleet's edges files could not all be read. try again", transient: true}

}

if ref := s.offlineSelect(ctx, fleet, host, mv.From, mv.To); ref != nil {

return offlineStep{}, ref

@@ -1194,7 +1198,7 @@ func (s *Server) offlineStepFor(ctx context.Context, d offlineDoor, mv *provisio

case provision.OfflineStopping:

fleet, ok := s.readOfflineFleet(ctx)

if !ok {

- return offlineStep{}, &offlineRefusal{status: http.StatusConflict, msg: "the fleet's edges files could not all be read. try again"}

+ return offlineStep{}, &offlineRefusal{status: http.StatusConflict, msg: "the fleet's edges files could not all be read. try again", transient: true}

}

sealer, boxes, ref := s.offlineSealCheck(fleet, ev, mv, host, from, now)

if ref != nil {

@@ -1230,7 +1234,7 @@ func (s *Server) offlineStepFor(ctx context.Context, d offlineDoor, mv *provisio

if mv.Sealer == "" {

fleet, ok := s.readOfflineFleet(ctx)

if !ok {

- return offlineStep{}, &offlineRefusal{status: http.StatusConflict, msg: "the fleet's edges files could not all be read. try again"}

+ return offlineStep{}, &offlineRefusal{status: http.StatusConflict, msg: "the fleet's edges files could not all be read. try again", transient: true}

}

boxes, sealerPhase = nil, ""

for _, b := range fleet.boxesAt(mv.From) {

diff --git a/internal/console/offlinereserve.go b/internal/console/offlinereserve.go

index 3f6a4c828..0463b5a0d 100644

--- a/internal/console/offlinereserve.go

+++ b/internal/console/offlinereserve.go

@@ -107,7 +107,7 @@ func offlineBusy(host string) *offlineRefusal {

}

var (

- offlineReserveUnreadable = &offlineRefusal{status: http.StatusBadGateway, msg: "could not read the instance"}

+ offlineReserveUnreadable = &offlineRefusal{status: http.StatusBadGateway, msg: "could not read the instance", transient: true}

offlineReserveMismatch = &offlineRefusal{status: http.StatusConflict, msg: "the reservation does not match this move's participants"}

)

diff --git a/internal/console/selfserve_checks_test.go b/internal/console/selfserve_checks_test.go

index 43fe0e56d..3152490ac 100644

--- a/internal/console/selfserve_checks_test.go

+++ b/internal/console/selfserve_checks_test.go

@@ -2,6 +2,7 @@ package console

import (

"testing"

+ "time"

"github.com/carlosframework/platform/internal/moves"

"github.com/carlosframework/platform/internal/provision"

@@ -110,3 +111,27 @@ func TestSelfServeDecisionsSpeakEachFact(t *testing.T) {

t.Error("the ready rule")

}

+// TestSelfServeTransientRefusalsAreTyped: the job waits out a refusal only

+// when the refusal itself says it clears on its own (built transient where

+// its sentence is), never by reading its text: a report not current, an

+// unread fleet, roster, manifest, claim or identity record, and the

+// source's facts not yet measured; a final refusal is not transient

+// whatever its words.

+func TestSelfServeTransientRefusalsAreTyped(t *testing.T) {

+ s := &Server{}

+ for _, ref := range []*offlineRefusal{offlineStaleRefusal("ie-box"), offlineReserveUnreadable} {

+ if !selfServeTransient(ref) {

+ t.Errorf("%q is not transient", ref.msg)

+ }

+ if _, ref := selfServeSourceFacts(nil, "shop.example.com", 1, time.Now()); ref == nil || !selfServeTransient(ref) {

+ t.Errorf("unmeasured facts: %+v", ref)

+ }

+ for _, ref := range []*offlineRefusal{s.selfServeOneBox("uk", 2), selfServeNotReady("ready"),

+ {status: 409, msg: "the fleet's edges files could not all be read. try again"}} {

+ if selfServeTransient(ref) {

+ t.Errorf("%q (built final) is transient", ref.msg)

+ }

diff --git a/internal/console/selfserve_door.go b/internal/console/selfserve_door.go

index 0f0257aa9..44084ed2e 100644

--- a/internal/console/selfserve_door.go

+++ b/internal/console/selfserve_door.go

@@ -188,7 +188,7 @@ func selfServeSourceFacts(m *status.BoxMetrics, host string, inc uint64, now tim

f, ok = m.Hosts[host]

}

if !ok {

- return f, &offlineRefusal{status: http.StatusConflict, msg: msgSelfServeUnmeasured}

+ return f, &offlineRefusal{status: http.StatusConflict, msg: msgSelfServeUnmeasured, transient: true}

}

fresh := func(at int64) bool {

age := now.Sub(time.Unix(at, 0))

@@ -196,11 +196,11 @@ func selfServeSourceFacts(m *status.BoxMetrics, host string, inc uint64, now tim

}

switch {

case f.Incarnation != inc || !fresh(f.CollectedAt) || !fresh(f.ReplicaCollectedAt):

- return f, &offlineRefusal{status: http.StatusConflict, msg: msgSelfServeSourceStale}

+ return f, &offlineRefusal{status: http.StatusConflict, msg: msgSelfServeSourceStale, transient: true}

case f.OverCap:

return f, &offlineRefusal{status: http.StatusConflict, msg: msgSelfServeTooLarge}

case !f.Known, f.Truncated:

- return f, &offlineRefusal{status: http.StatusConflict, msg: msgSelfServeUnmeasured}

+ return f, &offlineRefusal{status: http.StatusConflict, msg: msgSelfServeUnmeasured, transient: true}

}

return f, nil

}

diff --git a/internal/console/selfserve_job.go b/internal/console/selfserve_job.go

index aa0f00fd5..dd6498722 100644

--- a/internal/console/selfserve_job.go

+++ b/internal/console/selfserve_job.go

@@ -19,8 +19,6 @@ import (

"errors"

"fmt"

"log"

- "net/http"

- "strings"

"time"

"github.com/carlosframework/platform/internal/moves"

@@ -193,11 +191,8 @@ func selfServeWaiting(row *SelfServeMove, kinds ...string) bool {

// selfServeTransient reports whether a check's refusal is one that clears

// on its own (a read that failed, a report not yet current), which the job

-// waits out rather than aborting on.

-func selfServeTransient(ref *offlineRefusal) bool {

- return ref.status >= http.StatusInternalServerError || strings.Contains(ref.msg, "try again") ||

- strings.HasSuffix(ref.msg, "has not reported recently")

+// waits out rather than aborting on: the refusal says so where it is built.

+func selfServeTransient(ref *offlineRefusal) bool { return ref.transient }

// selfServeStep advances one row by at most one step, in the plan's order:

// (1) the row's deadlines, decided before any evidence is read; (2) the