carlos / platform Public

carlos instances create --from and --at, and the SEED column (Plan 5 Task 6)

The create posts, prints the copying line on the console's 202, then follows
the instance record every 2 seconds until the seed leaves reserving. An
aborted seed returns its reason as the error.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 2 weeks ago 248aa346ad185fe4866c51ff163547682c6c875c parent d4637d4

6 files changed, +438 −4

apimode.go +3 −0
approved_seeded_copy_test.go +3 −2
docs/site/cli.md +7 −0
instances_cli.go +23 −2
instances_seed.go +185 −0
instances_seed_test.go +217 −0

CI passed — run details

diff --git a/apimode.go b/apimode.go

index 86f988b07..ff08da522 100644

--- a/apimode.go

+++ b/apimode.go

@@ -3195,6 +3195,9 @@ type apiInstanceRecord struct {

// store afterwards is Environment, and `carlos store status` is what

// reads it.

PrivateStoreBucket string `json:"private_store_bucket"`

+ // Seed is the record's seed (seeded-instances spec §3.2); nil on an

+ // unseeded record and on an older console.

+ Seed *apiSeed `json:"seed,omitempty"`

// RegionName and RegionFrom say where the create put the record and how

// that was chosen (nearest-region §9). Empty from a console that predates

// them, and then the CLI prints today's line.

diff --git a/approved_seeded_copy_test.go b/approved_seeded_copy_test.go

index 80f1fd205..997f120c1 100644

--- a/approved_seeded_copy_test.go

+++ b/approved_seeded_copy_test.go

@@ -34,9 +34,10 @@ var seededCopyIDs = func() []string {

// seededWiredIDs grows as the plans wire rows. Plan 1 wires none; Plan 3

// Task 3 wires SD-N1, Task 5 SD-N2 to SD-N8 and SD-N113, Task 6 SD-N9 to SD-N13,

// SD-N111 and SD-N112; Plan 4 Task 3 SD-N14 to SD-N19; Plan 5 Task 3 SD-N27 to SD-N29,

-// Task 5 SD-N20 to SD-N26, SD-N30 to SD-N39 and SD-N41.

+// Task 5 SD-N20 to SD-N26, SD-N30 to SD-N39 and SD-N41; Plan 5 Task 6 SD-N40, SD-N42 to SD-N47, SD-N83 to SD-N88 and SD-N114.

var seededWiredIDs = []string{"SD-N1", "SD-N2", "SD-N3", "SD-N4", "SD-N5", "SD-N6", "SD-N7", "SD-N8", "SD-N113", "SD-N9", "SD-N10", "SD-N11", "SD-N12", "SD-N13", "SD-N111", "SD-N112", "SD-N14", "SD-N15", "SD-N16", "SD-N17", "SD-N18", "SD-N19", "SD-N27", "SD-N28", "SD-N29",

- "SD-N20", "SD-N21", "SD-N22", "SD-N23", "SD-N24", "SD-N25", "SD-N26", "SD-N30", "SD-N31", "SD-N32", "SD-N33", "SD-N34", "SD-N35", "SD-N36", "SD-N37", "SD-N38", "SD-N39", "SD-N41"}

+ "SD-N20", "SD-N21", "SD-N22", "SD-N23", "SD-N24", "SD-N25", "SD-N26", "SD-N30", "SD-N31", "SD-N32", "SD-N33", "SD-N34", "SD-N35", "SD-N36", "SD-N37", "SD-N38", "SD-N39", "SD-N41",

+ "SD-N40", "SD-N42", "SD-N43", "SD-N44", "SD-N45", "SD-N46", "SD-N47", "SD-N114", "SD-N83", "SD-N84", "SD-N85", "SD-N86", "SD-N87", "SD-N88"}

func mustParseSeededRecord(t *testing.T) []copyRow {

t.Helper()

diff --git a/docs/site/cli.md b/docs/site/cli.md

index 21c9988cb..e9c9c5a1c 100644

--- a/docs/site/cli.md

+++ b/docs/site/cli.md

@@ -694,6 +694,13 @@ holds the account's authoritative data, and until a batch of orders drains

home it holds order data in its own region — so the placement set is an

explicit decision on the home's record, never a default.

+To start a new instance from a copy of another instance's database, add --from. The copy comes from the source's replica, so the source keeps serving. Both instances belong to the same app and account, and the copy stays in the source's region; run carlos move afterwards to put it somewhere else. The new instance gets its own environment, never the source's, and an empty private store when the app uses one. Add --at with a transaction id to copy an earlier point in the source's history.

+```sh

+carlos instances create --app shop --host canary.example.com --from www.example.com

+carlos instances create --app shop --host canary.example.com --from www.example.com --at 4211

+```

#### What an instance is started with

A hibernating instance is started by the box when a request arrives for it,

diff --git a/instances_cli.go b/instances_cli.go

index 87a39de9c..a22dd0f64 100644

--- a/instances_cli.go

+++ b/instances_cli.go

@@ -301,6 +301,8 @@ func runInstancesCreate(args []string) {

acctRaw := fs.String("account", "", "account (sqid or name) the app belongs to (default: your only account, or the configured/chosen one)")

acctIDRaw := fs.String("account-id", "", "account sqid, matched as a sqid ONLY — supersedes --account")

role := fs.String("role", "", "instance role (default: the serving home); \"replica\" provisions a second instance of the account holding a one-way mirror of it, selling near the buyer — needs --region naming the replica's placement")

+ from := fs.String("from", "", "copy this instance's database into the new one. The source must be an instance of the same app, in this console's region")

+ at := fs.Int64("at", 0, "with --from, the transaction id to copy at (default: the latest)")

replicas := fs.String("replicas", "", "comma-separated placement codes the account's replicas live in (the HOME record's opt-in; e.g. ie,uk) — this instance's own replica must be --role replica with --region")

fs.Usage = flagUsage(fs, instancesRecordsUsage)

fs.Parse(args)

@@ -311,6 +313,12 @@ func runInstancesCreate(args []string) {

fs.Visit(func(f *flag.Flag) { set[f.Name] = true })

addrs := parseAddrList(*addr, set["addr"])

probe := parseProbeFlags(*healthPath, *healthStatus, set["health-path"], set["health-status"])

+ if set["at"] && (*at <= 0 || *from == "") {

+ log.Fatalf("instances create: %s", "at must be a transaction id, a whole number above 0")

+ }

+ if *from != "" && (*kind != "" || set["addr"] || *role != "" || *replicas != "") {

+ log.Fatalf("instances create: %s", "a seeded instance is an ordinary instance. Drop --kind, --addr and --role.")

+ }

sess := requireAPISession("create", *consoleFlag)

printUsing(stdout, sess)

acct := resolvedAccount(sess, acctRaw, acctIDRaw)

@@ -319,6 +327,13 @@ func runInstancesCreate(args []string) {

if merr != nil {

log.Fatalf("instances create: %v", merr)

}

+ if *from != "" {

+ if err := sess.runSeededCreate(context.Background(), stdout, acct, app2, strings.TrimSpace(*host), strings.TrimSpace(*region),

+ strings.TrimSpace(*channel), strings.TrimSpace(*environment), strings.TrimSpace(*from), uint64(*at), message); err != nil {

+ log.Fatalf("instances create: %v", err)

+ }

+ return

+ }

rec, err := sess.instanceCreate(context.Background(), acct, app2,

strings.TrimSpace(*host), strings.TrimSpace(*region), strings.TrimSpace(*channel),

strings.TrimSpace(*kind), strings.TrimSpace(*environment), addrs, probe,

@@ -334,6 +349,12 @@ func runInstancesCreate(args []string) {

mutedf(stdout, " the host comes live on the next reconcile pass on its box")

}

+// seedCopyingLine is what `instances create --from` says when the console

+// has reserved the record and the copy begins.

+func seedCopyingLine(source, host string) string {

+ return fmt.Sprintf("copying %s into %s. A large database can take several minutes.", source, host)

// instanceCreatedLine is the text after "created instance" or "canary

// host": today's "host (channel X<env>)" byte for byte from an older

// console, C14 when the console names the place.

@@ -435,13 +456,13 @@ func runInstancesList(args []string) {

log.Fatalf("instances list: %v", err)

}

w := newTable(stdout)

- fmt.Fprintln(w, "HOST\tREGION\tCHANNEL\tPOOL\tVERSION\tHEALTH\tUPSTREAMS\tDB\tCREATED\tBY")

+ fmt.Fprintln(w, "HOST\tREGION\tCHANNEL\tPOOL\tVERSION\tHEALTH\tUPSTREAMS\tDB\tCREATED\tBY\t"+"SEED")

for _, r := range recs {

rowHealth := r.Health

if rowHealth == "" {

rowHealth = "—" // an old console: the field predates it, not "unhealthy"

}

- fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", r.Host, r.Region, r.Channel, r.Pool, r.StagedVersion, rowHealth, upstreamsCell(r.Upstreams), humanBytes(r.DBBytes), r.CreatedAt, r.CreatedBy)

+ fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", r.Host, r.Region, r.Channel, r.Pool, r.StagedVersion, rowHealth, upstreamsCell(r.Upstreams), humanBytes(r.DBBytes), r.CreatedAt, r.CreatedBy, seedCell(r))

}

w.Flush()

if truncated {

diff --git a/instances_seed.go b/instances_seed.go

new file mode 100644

index 000000000..79391f5da

--- /dev/null

+++ b/instances_seed.go

@@ -0,0 +1,185 @@

+package main

+import (

+ "context"

+ "encoding/json"

+ "errors"

+ "fmt"

+ "io"

+ "net/http"

+ "strings"

+ "time"

+// apiSeed is the record's seed as the CLI renders it

+// (provision.SeedSource's wire shape).

+type apiSeed struct {

+ ID string `json:"id"`

+ Kind string `json:"kind"`

+ From string `json:"from,omitempty"`

+ TXID string `json:"txid,omitempty"`

+ SHA256 string `json:"sha256"`

+ Bytes int64 `json:"bytes"`

+ State string `json:"state"`

+ Reason string `json:"reason,omitempty"`

+// seedCell is the list's SEED column (seeded-instances spec §7).

+func seedCell(r apiInstanceRecord) string {

+ s := r.Seed

+ if s == nil {

+ return "-"

+ }

+ switch s.State {

+ case "reserving":

+ return fmt.Sprintf("copying from %s", s.From)

+ case "aborted", "rolled-back":

+ return "aborting"

+ case "refused":

+ return fmt.Sprintf("refused: %s", s.Reason)

+ }

+ if s.Kind != "import" {

+ return fmt.Sprintf("seeded from %s at txid %s", s.From, s.TXID)

+ }

+ switch s.State {

+ case "awaiting-upload":

+ return "import awaiting upload"

+ case "uploaded", "ready":

+ return "awaiting stop"

+ case "swapped", "placed":

+ return "swapped, awaiting verify"

+ }

+ sha := s.SHA256

+ if len(sha) > 12 {

+ sha = sha[:12]

+ }

+ return fmt.Sprintf("imported %s", sha)

+// Seams for the tests: how often the create follows the record, and how it

+// waits between looks.

+var (

+ seedPollEvery = 2 * time.Second

+ seedWait = func(ctx context.Context, d time.Duration) error {

+ t := time.NewTimer(d)

+ defer t.Stop()

+ select {

+ case <-t.C:

+ return nil

+ case <-ctx.Done():

+ return ctx.Err()

+ }

+// seedPollFailures is how many looks in a row may fail before the create

+// gives up and returns the last error.

+const seedPollFailures = 5

+// instanceCreateFrom is `carlos instances create --from` (seeded-instances

+// spec §4): the ordinary create body plus "from". A refusal before the

+// record is reserved comes back as an ordinary error. Once it is reserved

+// the console answers 202 with the reserving record and copies on its own;

+// that record is returned (accepted true) and the caller follows it with

+// followSeed. A console that answers 201 has finished the copy already.

+func (s *apiSession) instanceCreateFrom(ctx context.Context, a apiAcct, app, host, region, channel, environment, from string, at uint64, messages ...string) (rec apiInstanceRecord, accepted bool, err error) {

+ in := struct {

+ Host string `json:"host"`

+ Region string `json:"region,omitempty"`

+ Channel string `json:"channel,omitempty"`

+ Environment string `json:"environment,omitempty"`

+ Message string `json:"message,omitempty"`

+ From struct {

+ Host string `json:"host"`

+ TXID uint64 `json:"txid,omitempty"`

+ } `json:"from"`

+ }{Host: host, Region: region, Channel: channel, Environment: environment, Message: firstMutationMessage(messages)}

+ in.From.Host, in.From.TXID = from, at

+ status, body, err := s.postJSON(ctx, apiPath(a, app, "/instances"), in)

+ if err != nil {

+ return apiInstanceRecord{}, false, err

+ }

+ if status != http.StatusCreated && status != http.StatusAccepted {

+ return apiInstanceRecord{}, false, apiErrorFromBody(status, body)

+ }

+ if err := json.Unmarshal(body, &rec); err != nil {

+ return apiInstanceRecord{}, false, fmt.Errorf("instances create: unreadable response: %w", err)

+ }

+ return rec, status == http.StatusAccepted, nil

+// seedFailure is the error a finished-badly seed becomes: the console's own

+// reason, or its generic sentence for the source.

+func seedFailure(s *apiSeed, source string) error {

+ if s != nil && s.Reason != "" {

+ return errors.New(s.Reason)

+ }

+ return fmt.Errorf("could not copy %s. Nothing was created. Try again.", source)

+// followSeed polls the instance record every seedPollEvery until its seed

+// leaves reserving, and returns the record it last read. An aborted seed

+// returns its reason as the error. A failed look is tried again on the next

+// tick, up to seedPollFailures in a row; a record that has gone returns at

+// once. There is no overall timeout: the console's deadlines and its sweep

+// bring every seed to an end.

+func (s *apiSession) followSeed(ctx context.Context, a apiAcct, app string, first apiInstanceRecord) (apiInstanceRecord, error) {

+ source, id := strings.TrimSpace(first.Host), ""

+ if first.Seed != nil {

+ source, id = first.Seed.From, first.Seed.ID

+ }

+ failures := 0

+ for {

+ if err := seedWait(ctx, seedPollEvery); err != nil {

+ return apiInstanceRecord{}, err

+ }

+ rec, found, err := s.instanceGet(ctx, a, app, first.Host)

+ switch {

+ case err == nil && !found:

+ return apiInstanceRecord{}, notFoundErr()

+ case err != nil:

+ if failures++; failures >= seedPollFailures {

+ return apiInstanceRecord{}, err

+ }

+ continue

+ }

+ failures = 0

+ if rec.Seed == nil || rec.Seed.ID != id {

+ // The record no longer carries this seed.

+ return apiInstanceRecord{}, seedFailure(nil, source)

+ }

+ switch rec.Seed.State {

+ case "reserving":

+ continue

+ case "aborted", "rolled-back", "refused":

+ return apiInstanceRecord{}, seedFailure(rec.Seed, source)

+ }

+ return rec, nil

+ }

+// runSeededCreate is the --from create end to end: the post, the

+// copying line, the follow, and the success output. It returns the console's

+// refusal or the seed's reason as the error, for the caller to print.

+func (s *apiSession) runSeededCreate(ctx context.Context, w io.Writer, a apiAcct, app, host, region, channel, environment, from string, at uint64, message string) error {

+ rec, accepted, err := s.instanceCreateFrom(ctx, a, app, host, region, channel, environment, from, at, message)

+ if err != nil {

+ return err

+ }

+ if accepted {

+ fmt.Fprintln(w, seedCopyingLine(from, host))

+ final, err := s.followSeed(ctx, a, app, rec)

+ if err != nil {

+ return err

+ }

+ // The look at one record does not carry where the create placed it.

+ final.RegionName, final.RegionFrom = rec.RegionName, rec.RegionFrom

+ rec = final

+ }

+ fmt.Fprintf(w, "%s %s\n", colorFor(w).ok("created instance"), instanceCreatedLine(rec, environmentSuffix(rec.Environment)))

+ if rec.Seed != nil {

+ mutedf(w, " %s", seedCell(rec))

+ }

+ mutedf(w, " the host comes live on the next reconcile pass on its box")

+ return nil

diff --git a/instances_seed_test.go b/instances_seed_test.go

new file mode 100644

index 000000000..e2672fc9c

--- /dev/null

+++ b/instances_seed_test.go

@@ -0,0 +1,217 @@

+package main

+import (

+ "bytes"

+ "context"

+ "encoding/json"

+ "errors"

+ "fmt"

+ "io"

+ "net/http"

+ "net/http/httptest"

+ "strings"

+ "sync/atomic"

+ "testing"

+ "time"

+func TestSeedCell(t *testing.T) {

+ for _, c := range []struct {

+ seed *apiSeed

+ want string

+ }{

+ {nil, "-"},

+ {&apiSeed{Kind: "replica", State: "reserving", From: "www.example.com"}, "copying from www.example.com"},

+ {&apiSeed{Kind: "replica", State: "ready", From: "www.example.com", TXID: "4211"}, "seeded from www.example.com at txid 4211"},

+ {&apiSeed{Kind: "replica", State: "placed", From: "www.example.com", TXID: "4211"}, "seeded from www.example.com at txid 4211"},

+ {&apiSeed{Kind: "import", State: "awaiting-upload"}, "import awaiting upload"},

+ {&apiSeed{Kind: "import", State: "uploaded"}, "awaiting stop"},

+ {&apiSeed{Kind: "import", State: "ready"}, "awaiting stop"},

+ {&apiSeed{Kind: "import", State: "swapped"}, "swapped, awaiting verify"},

+ {&apiSeed{Kind: "import", State: "committed", SHA256: "0123456789abcdef0123"}, "imported 0123456789ab"},

+ {&apiSeed{Kind: "import", State: "refused", Reason: "the route belongs to app kass"}, "refused: the route belongs to app kass"},

+ {&apiSeed{Kind: "import", State: "aborted"}, "aborting"},

+ {&apiSeed{Kind: "replica", State: "rolled-back"}, "aborting"},

+ } {

+ if got := seedCell(apiInstanceRecord{Seed: c.seed}); got != c.want {

+ t.Errorf("seedCell(%+v) = %q, want %q", c.seed, got, c.want)

+ }

+func TestInstanceCreateFromSendsTheSource(t *testing.T) {

+ var body map[string]any

+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ raw, _ := io.ReadAll(r.Body)

+ _ = json.Unmarshal(raw, &body)

+ w.WriteHeader(http.StatusAccepted)

+ w.Write([]byte(`{"host":"canary.example.com","channel":"stable","seed":{"id":"sd_x","kind":"replica","from":"www.example.com","state":"reserving"}}`))

+ }))

+ defer ts.Close()

+ rec, accepted, err := newTestSession(ts.URL).instanceCreateFrom(context.Background(), apiAcct{Seg: "bab"}, "shop",

+ "canary.example.com", "", "", "", "www.example.com", 4200)

+ if err != nil {

+ t.Fatal(err)

+ }

+ from, _ := body["from"].(map[string]any)

+ if from["host"] != "www.example.com" || from["txid"] != float64(4200) || body["host"] != "canary.example.com" {

+ t.Fatalf("body = %v", body)

+ }

+ if !accepted || rec.Seed == nil || rec.Seed.State != "reserving" {

+ t.Fatalf("accepted = %v, record = %+v", accepted, rec)

+ }

+// seedConsole is a console whose create answers 202 and whose record then

+// walks through the given GET answers, one per look.

+type seedConsole struct {

+ createStatus int

+ createBody string

+ gets []func(w http.ResponseWriter)

+ looks atomic.Int32

+func (c *seedConsole) ServeHTTP(w http.ResponseWriter, r *http.Request) {

+ if r.Method == http.MethodPost {

+ w.WriteHeader(c.createStatus)

+ io.WriteString(w, c.createBody)

+ return

+ }

+ i := int(c.looks.Add(1)) - 1

+ if i >= len(c.gets) {

+ i = len(c.gets) - 1

+ }

+ c.gets[i](w)

+func seedRecord(state, reason string) func(w http.ResponseWriter) {

+ return func(w http.ResponseWriter) {

+ fmt.Fprintf(w, `{"host":"canary.example.com","channel":"stable","environment":"canary-env","seed":{"id":"sd_x","kind":"replica","from":"www.example.com","txid":"4211","state":%q,"reason":%q}}`, state, reason)

+ }

+func answerWith(code int, body string) func(w http.ResponseWriter) {

+ return func(w http.ResponseWriter) {

+ w.WriteHeader(code)

+ io.WriteString(w, body)

+ }

+const reservingBody = `{"host":"canary.example.com","channel":"stable","region_name":"Ireland","seed":{"id":"sd_x","kind":"replica","from":"www.example.com","state":"reserving"}}`

+// runSeeded runs a --from create against c with the wait faked, returning

+// the output, the error and the sleeps taken.

+func runSeeded(t *testing.T, c *seedConsole) (string, error, []time.Duration) {

+ t.Helper()

+ ts := httptest.NewServer(c)

+ defer ts.Close()

+ var waits []time.Duration

+ oldWait, oldEvery := seedWait, seedPollEvery

+ seedWait = func(ctx context.Context, d time.Duration) error { waits = append(waits, d); return ctx.Err() }

+ seedPollEvery = 2 * time.Second

+ defer func() { seedWait, seedPollEvery = oldWait, oldEvery }()

+ var out bytes.Buffer

+ err := newTestSession(ts.URL).runSeededCreate(context.Background(), &out, apiAcct{Seg: "bab"}, "shop",

+ "canary.example.com", "", "", "", "www.example.com", 0, "")

+ return out.String(), err, waits

+func TestSeededCreateFollowsTheRecordToReady(t *testing.T) {

+ out, err, waits := runSeeded(t, &seedConsole{createStatus: 202, createBody: reservingBody,

+ gets: []func(http.ResponseWriter){seedRecord("reserving", ""), seedRecord("reserving", ""), seedRecord("ready", "")}})

+ if err != nil {

+ t.Fatal(err)

+ }

+ if !strings.HasPrefix(out, "copying www.example.com into canary.example.com. A large database can take several minutes.\ncreated instance canary.example.com") ||

+ !strings.Contains(out, "seeded from www.example.com at txid 4211") ||

+ !strings.Contains(out, "the host comes live on the next reconcile pass on its box") {

+ t.Fatalf("output = %q", out)

+ }

+ if len(waits) != 3 || waits[0] != 2*time.Second {

+ t.Fatalf("waits = %v, want three waits of 2s", waits)

+ }

+func TestSeededCreateAbortedReturnsTheReason(t *testing.T) {

+ const reason = "copying www.example.com took longer than 30 minutes, so no instance was created."

+ out, err, _ := runSeeded(t, &seedConsole{createStatus: 202, createBody: reservingBody,

+ gets: []func(http.ResponseWriter){seedRecord("reserving", ""), seedRecord("aborted", reason)}})

+ if err == nil || err.Error() != reason {

+ t.Fatalf("err = %v, want exactly %q", err, reason)

+ }

+ if strings.Contains(out, "created instance") {

+ t.Fatalf("output = %q", out)

+ }

+func TestSeededCreateAbortedWithoutAReasonNamesTheSource(t *testing.T) {

+ _, err, _ := runSeeded(t, &seedConsole{createStatus: 202, createBody: reservingBody,

+ gets: []func(http.ResponseWriter){seedRecord("aborted", "")}})

+ if err == nil || err.Error() != "could not copy www.example.com. Nothing was created. Try again." {

+ t.Fatalf("err = %v", err)

+ }

+func TestSeededCreateRefusalPrintsLikeAnyCreateRefusal(t *testing.T) {

+ out, err, waits := runSeeded(t, &seedConsole{createStatus: 409, createBody: `{"error":"instance canary.example.com already exists"}`})

+ if err == nil || err.Error() != "instance canary.example.com already exists" {

+ t.Fatalf("err = %v", err)

+ }

+ if out != "" || len(waits) != 0 {

+ t.Fatalf("a refusal says nothing and waits for nothing: out = %q, waits = %v", out, waits)

+ }

+func TestSeededCreateRetriesATransientPollError(t *testing.T) {

+ out, err, waits := runSeeded(t, &seedConsole{createStatus: 202, createBody: reservingBody,

+ gets: []func(http.ResponseWriter){answerWith(502, `{"error":"could not read the instance"}`), answerWith(503, ""),

+ seedRecord("reserving", ""), answerWith(502, ""), seedRecord("placed", "")}})

+ if err != nil {

+ t.Fatal(err)

+ }

+ if !strings.Contains(out, "created instance canary.example.com") || len(waits) != 5 {

+ t.Fatalf("out = %q, waits = %v", out, waits)

+ }

+func TestSeededCreateGivesUpAfterFiveFailedLooks(t *testing.T) {

+ c := &seedConsole{createStatus: 202, createBody: reservingBody,

+ gets: []func(http.ResponseWriter){answerWith(502, `{"error":"could not read the instance"}`)}}

+ _, err, _ := runSeeded(t, c)

+ if err == nil || err.Error() != "could not read the instance" {

+ t.Fatalf("err = %v", err)

+ }

+ if n := c.looks.Load(); n != 5 {

+ t.Fatalf("looks = %d, want 5", n)

+ }

+func TestSeededCreateFailedLooksMustBeConsecutive(t *testing.T) {

+ bad := answerWith(502, "")

+ _, err, _ := runSeeded(t, &seedConsole{createStatus: 202, createBody: reservingBody,

+ gets: []func(http.ResponseWriter){bad, bad, bad, bad, seedRecord("reserving", ""), bad, bad, bad, bad, seedRecord("ready", "")}})

+ if err != nil {

+ t.Fatalf("four failures, a good look, four more: err = %v", err)

+ }

+func TestSeededCreateAMissingRecordEndsAtOnce(t *testing.T) {

+ c := &seedConsole{createStatus: 202, createBody: reservingBody, gets: []func(http.ResponseWriter){answerWith(404, "")}}

+ _, err, _ := runSeeded(t, c)

+ if err == nil || err.Error() != notFoundErr().Error() || c.looks.Load() != 1 {

+ t.Fatalf("err = %v, looks = %d", err, c.looks.Load())

+ }

+func TestSeededCreateStopsWhenTheContextEnds(t *testing.T) {

+ ts := httptest.NewServer(&seedConsole{createStatus: 202, createBody: reservingBody, gets: []func(http.ResponseWriter){seedRecord("reserving", "")}})

+ defer ts.Close()

+ ctx, cancel := context.WithCancel(context.Background())

+ oldWait := seedWait

+ seedWait = func(ctx context.Context, d time.Duration) error { cancel(); return ctx.Err() }

+ defer func() { seedWait = oldWait }()

+ err := newTestSession(ts.URL).runSeededCreate(ctx, io.Discard, apiAcct{Seg: "bab"}, "shop", "canary.example.com", "", "", "", "www.example.com", 0, "")

+ if !errors.Is(err, context.Canceled) {

+ t.Fatalf("err = %v", err)

+ }