| index c31b83727..62d9c4427 100644 |
| --- a/internal/console/apiinstanceimport_test.go |
| +++ b/internal/console/apiinstanceimport_test.go |
| @@ -12,6 +12,7 @@ import ( |
| "net/http" |
| "os" |
| "path/filepath" |
| + "strconv" |
| "strings" |
| "sync" |
| "sync/atomic" |
| @@ -1265,6 +1266,12 @@ func (s *movesOnWrite) Put(ctx context.Context, key string, body []byte, ifMatch |
| return s.Store.Put(ctx, key, body, ifMatch) |
| } |
| |
| +// DeleteIfMatch forwards, so a finish behind the wrapper can delete the |
| +// record. |
| +func (s *movesOnWrite) DeleteIfMatch(ctx context.Context, key, etag string) error { |
| + return release.DeleteIfMatch(ctx, s.Store, key, etag) |
| +} |
| + |
| // A stopped or a commit that read the import in one state and finds it in |
| // another when it writes changes nothing: SD-N60, no audit row and no bump. |
| // It never acts on an import that has moved on (its transition is guarded |
| @@ -1395,3 +1402,548 @@ func TestImportOverlappingTransitions(t *testing.T) { |
| }) |
| } |
| } |
| + |
| +// ---- Abort and the delete door (spec §5.6, preflight M8, M14, M15) ---- |
| + |
| +const ( |
| + mainDeleteFailed = "could not delete the instance — try again" |
| + importRunBusy = "carlos run is busy on " + importHost + ". Wait for it to finish, then abort." |
| +) |
| + |
| +func importChanged(host string) string { |
| + return host + " changed while this request was made. Read its state with carlos instances list and try again." |
| +} |
| + |
| +func importCommitted(host string) string { |
| + return host + " is committed. Delete it with carlos instances delete instead." |
| +} |
| + |
| +func importAbortFirst(host string) string { |
| + return host + " is an import that is not committed. Abort the import instead." |
| +} |
| + |
| +// postImportErr is postImport for a goroutine: it returns its error and |
| +// never calls t.Fatal (preflight m9). |
| +func postImportErr(f fromFixture, host, verb string) (int, map[string]any, error) { |
| + req, err := http.NewRequest(http.MethodPost, f.url+"/"+host+"/import/"+verb, nil) |
| + if err != nil { |
| + return 0, nil, err |
| + } |
| + req.Header.Set("Authorization", "Bearer "+f.token) |
| + res, err := http.DefaultClient.Do(req) |
| + if err != nil { |
| + return 0, nil, err |
| + } |
| + defer res.Body.Close() |
| + var out map[string]any |
| + _ = json.NewDecoder(res.Body).Decode(&out) |
| + return res.StatusCode, out, nil |
| +} |
| + |
| +// deleteInstance sends the ordinary delete for host and returns what it |
| +// answered. |
| +func deleteInstance(t *testing.T, f fromFixture, host string) (int, map[string]any) { |
| + t.Helper() |
| + code, _, raw := rawReq(t, http.MethodDelete, f.url+"/"+host, f.token, "") |
| + var out map[string]any |
| + _ = json.Unmarshal([]byte(raw), &out) |
| + return code, out |
| +} |
| + |
| +// setRun writes the run series' field directly, through the public |
| +// etag-guarded writer: run admission refuses an uncommitted import, so a |
| +// run open on one is only reachable by a write like this (a run admitted |
| +// before the import began, in the spec's words). |
| +func setRun(t *testing.T, f fromFixture, host, state string) { |
| + t.Helper() |
| + rec, etag, err := provision.GetRecordETag(context.Background(), f.fx.rel, "bab", fromApp, host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + rec.ActiveRun = &provision.RunState{ID: "run_1", State: state, Deadline: time.Now().Add(time.Hour)} |
| + if err := provision.UpdateRecord(context.Background(), f.fx.rel, *rec, etag); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// A commit and an abort race on the record's etag, and exactly one wins |
| +// (spec §5.6). The loser is told why: an abort that lost to the commit is |
| +// SD-N56, a commit that lost to the abort is SD-N60 (or SD-N115 when the |
| +// abort landed before its read). Neither is a 502. |
| +func TestImportCommitAbortRace(t *testing.T) { |
| + f := newImportFixture(t) |
| + body, _ := sqliteBytes(t) |
| + begun := f.begun(t, importHost, body) |
| + requireImportState(t, f.uploaded(t, importHost, body), begun.Seed.ID, seeds.StateUploaded, "") |
| + if code, out := postImport(t, f, importHost, "stopped"); code != http.StatusOK { |
| + t.Fatalf("stopped = %d %v", code, out) |
| + } |
| + f.setSeed(t, provision.SeedSwapped(begun.Seed.ID)) |
| + f.setSeed(t, provision.SeedPlace(begun.Seed.ID)) |
| + hosts := []string{importHost} |
| + for i := range 8 { |
| + host := "race" + strconv.Itoa(i) + ".shop.example.com" |
| + f.seedRecord(t, host, "", newSeedID(t), seeds.KindImport, seeds.StatePlaced) |
| + hosts = append(hosts, host) |
| + } |
| + type answer struct { |
| + verb string |
| + code int |
| + out map[string]any |
| + err error |
| + } |
| + for _, host := range hosts { |
| + answers := make(chan answer, 2) |
| + for _, verb := range []string{"commit", "abort"} { |
| + go func(verb string) { |
| + code, out, err := postImportErr(f, host, verb) |
| + answers <- answer{verb, code, out, err} |
| + }(verb) |
| + } |
| + got := map[string]answer{} |
| + for range 2 { |
| + a := <-answers |
| + if a.err != nil { |
| + t.Fatalf("%s %s: %v", host, a.verb, a.err) |
| + } |
| + got[a.verb] = a |
| + } |
| + rec, err := f.record(t, host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + commit, abort := got["commit"], got["abort"] |
| + switch rec.Seed.State { |
| + case seeds.StateCommitted: |
| + if commit.code != http.StatusOK || abort.code != http.StatusConflict || abort.out["error"] != importCommitted(host) { |
| + t.Fatalf("%s committed: commit %d %v, abort %d %v", host, commit.code, commit.out, abort.code, abort.out) |
| + } |
| + case seeds.StateAborted: |
| + lost := commit.out["error"] == importChanged(host) || commit.out["error"] == host+" is aborted, so there is nothing to commit." |
| + if abort.code != http.StatusOK || abort.out["state"] != seeds.StateAborted || commit.code != http.StatusConflict || !lost { |
| + t.Fatalf("%s aborted: commit %d %v, abort %d %v", host, commit.code, commit.out, abort.code, abort.out) |
| + } |
| + default: |
| + t.Fatalf("%s ended %s: commit %d %v, abort %d %v; want exactly one winner", host, rec.Seed.State, commit.code, commit.out, abort.code, abort.out) |
| + } |
| + } |
| + committed, aborted := importAudit(t, f, "import-committed"), importAudit(t, f, "import-aborted") |
| + if len(committed)+len(aborted) != len(hosts) { |
| + t.Fatalf("audit rows: %d committed, %d aborted, want one per host (%d)", len(committed), len(aborted), len(hosts)) |
| + } |
| +} |
| + |
| +// Abort is refused while a run is open and after commit, lands from every |
| +// state before commit, and is safe to repeat. |
| +func TestImportAbortRefusals(t *testing.T) { |
| + f := newImportFixture(t) |
| + body, sha := sqliteBytes(t) |
| + code, out := beginImport(t, f, importHost, sha, int64(len(body))) |
| + begun, _ := f.requireBegun(t, code, out, importHost) |
| + id := begun.Seed.ID |
| + |
| + setRun(t, f, importHost, provision.RunPicked) |
| + before := serial(t, f.fx) |
| + if code, out := postImport(t, f, importHost, "abort"); code != http.StatusConflict || str(t, out, "error") != importRunBusy { |
| + t.Fatalf("abort with a run open = %d %v", code, out) |
| + } |
| + rec, _ := f.record(t, importHost) |
| + requireImportState(t, rec, id, seeds.StateAwaitingUpload, "") |
| + if serial(t, f.fx) != before || len(importAudit(t, f, "import-aborted")) != 0 { |
| + t.Fatal("a refused abort bumped the serial or wrote an audit row") |
| + } |
| + |
| + setRun(t, f, importHost, provision.RunDone) |
| + if code, out := postImport(t, f, importHost, "abort"); code != http.StatusOK || len(out) != 1 || out["state"] != "aborted" { |
| + t.Fatalf("abort = %d %v", code, out) |
| + } |
| + rec, _ = f.record(t, importHost) |
| + requireImportState(t, rec, id, seeds.StateAborted, "") |
| + if got := serial(t, f.fx); got != before+1 { |
| + t.Fatalf("serial after abort = %d, want %d", got, before+1) |
| + } |
| + if rows := importAudit(t, f, "import-aborted"); len(rows) != 1 || rows[0] != "aborted the import of "+importHost { |
| + t.Fatalf("import-aborted rows = %q", rows) |
| + } |
| + |
| + // Again: 200, and nothing written, audited or bumped. |
| + before = serial(t, f.fx) |
| + if code, out := postImport(t, f, importHost, "abort"); code != http.StatusOK || len(out) != 1 || out["state"] != "aborted" { |
| + t.Fatalf("abort again = %d %v, want 200", code, out) |
| + } |
| + if serial(t, f.fx) != before || len(importAudit(t, f, "import-aborted")) != 1 { |
| + t.Fatal("a repeated abort bumped the serial or wrote an audit row") |
| + } |
| + |
| + // After commit, abort is refused: delete is the verb then. |
| + f.seedRecord(t, "done.shop.example.com", "", newSeedID(t), seeds.KindImport, seeds.StateCommitted) |
| + if code, out := postImport(t, f, "done.shop.example.com", "abort"); code != http.StatusConflict || |
| + str(t, out, "error") != importCommitted("done.shop.example.com") { |
| + t.Fatalf("abort after commit = %d %v", code, out) |
| + } |
| + if rec, _ := f.record(t, "done.shop.example.com"); rec.Seed.State != seeds.StateCommitted { |
| + t.Fatalf("a refused abort moved a committed import to %s", rec.Seed.State) |
| + } |
| +} |
| + |
| +// Every state before committed aborts. A refused import keeps the reason |
| +// the box gave (preflight M14: the no-reason SeedAbort). |
| +func TestImportAbortFromEveryStateBeforeCommit(t *testing.T) { |
| + f := newImportFixture(t) |
| + for _, state := range []string{seeds.StateAwaitingUpload, seeds.StateUploaded, seeds.StateReady, |
| + seeds.StateSwapped, seeds.StatePlaced, seeds.StateRefused} { |
| + host := state + ".shop.example.com" |
| + id := newSeedID(t) |
| + f.seedRecord(t, host, "", id, seeds.KindImport, state) |
| + reason := "" |
| + if state == seeds.StateRefused { |
| + reason = "the box found no addr row" |
| + rec, etag, err := provision.GetRecordETag(context.Background(), f.fx.rel, "bab", fromApp, host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + rec.Seed.Reason = reason |
| + raw, _ := json.Marshal(rec) |
| + if _, err := f.fx.rel.Put(context.Background(), provision.RecordKey("bab", fromApp, host), raw, etag); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if code, out := postImport(t, f, host, "abort"); code != http.StatusOK || out["state"] != "aborted" { |
| + t.Errorf("abort while %s = %d %v", state, code, out) |
| + continue |
| + } |
| + rec, _ := f.record(t, host) |
| + requireImportState(t, rec, id, seeds.StateAborted, reason) |
| + } |
| +} |
| + |
| +// A rolled-back import is finished by its abort: the record and the |
| +// snapshot go, and the answer says rolled-back. A finish that fails still |
| +// answers 200 (the abort has landed), and leaves the record rolled-back |
| +// for the next abort or the sweep. |
| +func TestImportAbortFinishesARolledBackImport(t *testing.T) { |
| + f := newImportFixture(t) |
| + body, _ := sqliteBytes(t) |
| + begun := f.begun(t, importHost, body) |
| + id := begun.Seed.ID |
| + requireImportState(t, f.uploaded(t, importHost, body), id, seeds.StateUploaded, "") |
| + f.setSeed(t, provision.SeedAbort(id)) |
| + f.setSeed(t, provision.SeedRolledBack(id)) |
| + |
| + data := f.fx.srv.cfg.SeedData |
| + f.fx.srv.cfg.SeedData = nil // the finish cannot delete the snapshot |
| + if code, out := postImport(t, f, importHost, "abort"); code != http.StatusOK || len(out) != 1 || out["state"] != "rolled-back" { |
| + t.Fatalf("abort with a failing finish = %d %v", code, out) |
| + } |
| + rec, _ := f.record(t, importHost) |
| + requireImportState(t, rec, id, seeds.StateRolledBack, "") |
| + |
| + f.fx.srv.cfg.SeedData = data |
| + if code, out := postImport(t, f, importHost, "abort"); code != http.StatusOK || len(out) != 1 || out["state"] != "rolled-back" { |
| + t.Fatalf("abort of a rolled-back import = %d %v", code, out) |
| + } |
| + f.requireRecordGone(t, importHost) |
| + f.requireNothingUploaded(t) |
| + if rows := importAudit(t, f, "import-aborted"); len(rows) != 0 { |
| + t.Fatalf("finishing a rolled-back import audited an abort: %q", rows) |
| + } |
| +} |
| + |
| +// An abort that loses its race re-reads once and answers by what it finds |
| +// (preflight M14): another abort is a 200, a commit is SD-N56, a rollback |
| +// is finished and a 200, a run that opened is SD-N58, and anything else is |
| +// SD-N60. The loser writes no audit row and bumps nothing. |
| +func TestImportAbortLosesARace(t *testing.T) { |
| + ctx := context.Background() |
| + for _, c := range []struct { |
| + name string |
| + hook func(f fromFixture, id string) error |
| + code int |
| + state string // the 200's state, or "" for an error |
| + msg string |
| + after string // the record's state after, "" for gone |
| + }{ |
| + {"to another abort", func(f fromFixture, id string) error { |
| + _, err := provision.SetRecordSeed(ctx, f.fx.rel, "bab", fromApp, importHost, provision.SeedAbort(id)) |
| + return err |
| + }, http.StatusOK, seeds.StateAborted, "", seeds.StateAborted}, |
| + {"to a commit", func(f fromFixture, id string) error { |
| + _, err := provision.SetRecordSeed(ctx, f.fx.rel, "bab", fromApp, importHost, provision.SeedCommit(id)) |
| + return err |
| + }, http.StatusConflict, "", importCommitted(importHost), seeds.StateCommitted}, |
| + {"to a rollback", func(f fromFixture, id string) error { |
| + for _, step := range []func(string) func(*provision.Record) error{provision.SeedAbort, provision.SeedRolledBack} { |
| + if _, err := provision.SetRecordSeed(ctx, f.fx.rel, "bab", fromApp, importHost, step(id)); err != nil { |
| + return err |
| + } |
| + } |
| + return nil |
| + }, http.StatusOK, seeds.StateRolledBack, "", ""}, |
| + {"to a run", func(f fromFixture, id string) error { |
| + rec, etag, err := provision.GetRecordETag(ctx, f.fx.rel, "bab", fromApp, importHost) |
| + if err != nil { |
| + return err |
| + } |
| + rec.ActiveRun = &provision.RunState{ID: "run_1", State: provision.RunQueued, Deadline: time.Now().Add(time.Hour)} |
| + return provision.UpdateRecord(ctx, f.fx.rel, *rec, etag) |
| + }, http.StatusConflict, "", importRunBusy, seeds.StatePlaced}, |
| + {"to another import of the host", func(f fromFixture, id string) error { |
| + if err := f.fx.rel.Delete(ctx, provision.RecordKey("bab", fromApp, importHost)); err != nil { |
| + return err |
| + } |
| + other, err := seeds.NewID() |
| + if err != nil { |
| + return err |
| + } |
| + rec := provision.Record{Schema: 1, Host: importHost, Slug: "old", App: fromApp, Account: "bab", Channel: "stable", CreatedAt: time.Now().UTC(), |
| + Seed: &provision.SeedSource{ID: other, Kind: seeds.KindImport, State: seeds.StateAborted, At: time.Now().UTC()}} |
| + return provision.CreateRecord(ctx, f.fx.rel, rec) |
| + }, http.StatusConflict, "", importChanged(importHost), seeds.StateAborted}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + f := newImportFixture(t) |
| + id := newSeedID(t) |
| + f.seedRecord(t, importHost, "", id, seeds.KindImport, seeds.StatePlaced) |
| + f.fx.srv.cfg.Releases = &movesOnWrite{Store: f.fx.rel, key: provision.RecordKey("bab", fromApp, importHost), hook: func() { |
| + if err := c.hook(f, id); err != nil { |
| + t.Error(err) |
| + } |
| + }} |
| + before := serial(t, f.fx) |
| + code, out := postImport(t, f, importHost, "abort") |
| + if code != c.code || (c.state != "" && (len(out) != 1 || out["state"] != c.state)) || (c.msg != "" && str(t, out, "error") != c.msg) { |
| + t.Fatalf("abort = %d %v, want %d %q %q", code, out, c.code, c.state, c.msg) |
| + } |
| + rec, err := f.record(t, importHost) |
| + switch { |
| + case c.after == "": |
| + f.requireRecordGone(t, importHost) |
| + case err != nil || rec.Seed == nil || rec.Seed.State != c.after: |
| + t.Fatalf("record after = %+v, %v; want %s", rec, err, c.after) |
| + } |
| + if c.after != "" && serial(t, f.fx) != before { |
| + t.Fatal("the losing abort bumped the serial") |
| + } |
| + if rows := importAudit(t, f, "import-aborted"); len(rows) != 0 { |
| + t.Fatalf("the losing abort audited: %q", rows) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// Abort shares the import gate. |
| +func TestImportAbortGate(t *testing.T) { |
| + f := newImportFixture(t) |
| + f.seedRecord(t, fromHost, "", newSeedID(t), seeds.KindReplica, seeds.StateReady) |
| + f.seedRecord(t, importHost, "", newSeedID(t), seeds.KindImport, seeds.StatePlaced) |
| + for _, c := range []struct { |
| + name, host string |
| + code int |
| + msg string |
| + }{ |
| + {"no record", "ghost.shop.example.com", http.StatusNotFound, "ghost.shop.example.com is not an instance of shop"}, |
| + {"not a host", "Not_A_Host", http.StatusNotFound, "Not_A_Host is not an instance of shop"}, |
| + {"an ordinary instance", fromSource, http.StatusConflict, fromSource + " is not an import."}, |
| + {"a --from seed", fromHost, http.StatusConflict, fromHost + " is not an import."}, |
| + } { |
| + if code, out := postImport(t, f, c.host, "abort"); code != c.code || str(t, out, "error") != c.msg { |
| + t.Errorf("abort %s = %d %v, want %d %q", c.name, code, out, c.code, c.msg) |
| + } |
| + } |
| + f.fx.srv.cfg.HostClaim = true |
| + if code, out := postImport(t, f, importHost, "abort"); code != http.StatusConflict || |
| + str(t, out, "error") != seedClaimConsoleRefusal().Msg { |
| + t.Errorf("abort with claims armed = %d %v", code, out) |
| + } |
| + f.fx.srv.cfg.HostClaim = false |
| + if rec, _ := f.record(t, importHost); rec.Seed.State != seeds.StatePlaced { |
| + t.Fatalf("a refused abort moved the import to %s", rec.Seed.State) |
| + } |
| +} |
| + |
| +// The delete door's import rule (spec §5.6, codex R2-7): an import no box |
| +// has swapped is rolled back and finished by the delete, answered as the |
| +// ordinary delete is; a rolled-back one is finished; every other |
| +// uncommitted import is refused with SD-N57; a committed one is an |
| +// ordinary instance. |
| +func TestImportDeleteRefusedUntilCommittedOrUnswapped(t *testing.T) { |
| + f := newImportFixture(t) |
| + body, sha := sqliteBytes(t) |
| + code, out := beginImport(t, f, "early.shop.example.com", sha, int64(len(body))) |
| + f.requireBegun(t, code, out, "early.shop.example.com") |
| + before := serial(t, f.fx) |
| + if code, out := deleteInstance(t, f, "early.shop.example.com"); code != http.StatusOK || len(out) != 1 || out["result"] != "deleted" { |
| + t.Fatalf("delete before upload = %d %v, want 200 deleted", code, out) |
| + } |
| + f.requireRecordGone(t, "early.shop.example.com") |
| + if serial(t, f.fx) <= before { |
| + t.Fatal("the delete did not bump the serial") |
| + } |
| + if rows := importAudit(t, f, "instance-delete"); len(rows) != 1 || rows[0] != "deleted instance early.shop.example.com (cli)" { |
| + t.Fatalf("instance-delete rows = %q", rows) |
| + } |
| + // Again: the ordinary delete's idempotent 200. |
| + if code, out := deleteInstance(t, f, "early.shop.example.com"); code != http.StatusOK || out["result"] != "deleted" { |
| + t.Fatalf("second delete = %d %v", code, out) |
| + } |
| + |
| + // Uploaded: the snapshot goes too. |
| + begun := f.begun(t, importHost, body) |
| + requireImportState(t, f.uploaded(t, importHost, body), begun.Seed.ID, seeds.StateUploaded, "") |
| + if keys, _ := f.data.List(context.Background(), seeds.Prefix); len(keys) != 1 { |
| + t.Fatalf("seeds/ before the delete = %v, want the snapshot", keys) |
| + } |
| + if code, out := deleteInstance(t, f, importHost); code != http.StatusOK || out["result"] != "deleted" { |
| + t.Fatalf("delete after upload = %d %v", code, out) |
| + } |
| + f.requireRecordGone(t, importHost) |
| + f.requireNothingUploaded(t) |
| + |
| + // Refused (no box swapped) and rolled-back are finished; aborted, |
| + // ready, swapped and placed are the box's to roll back first. |
| + for _, c := range []struct { |
| + state string |
| + gone bool |
| + }{ |
| + {seeds.StateRefused, true}, |
| + {seeds.StateRolledBack, true}, |
| + {seeds.StateReady, false}, |
| + {seeds.StateSwapped, false}, |
| + {seeds.StatePlaced, false}, |
| + {seeds.StateAborted, false}, |
| + } { |
| + host := c.state + ".shop.example.com" |
| + f.seedRecord(t, host, "", newSeedID(t), seeds.KindImport, c.state) |
| + code, out := deleteInstance(t, f, host) |
| + if c.gone { |
| + if code != http.StatusOK || out["result"] != "deleted" { |
| + t.Errorf("delete while %s = %d %v, want 200", c.state, code, out) |
| + } |
| + f.requireRecordGone(t, host) |
| + continue |
| + } |
| + if code != http.StatusConflict || str(t, out, "error") != importAbortFirst(host) { |
| + t.Errorf("delete while %s = %d %v, want 409 %q", c.state, code, out, importAbortFirst(host)) |
| + } |
| + if rec, err := f.record(t, host); err != nil || rec.Seed.State != c.state { |
| + t.Errorf("a refused delete while %s left %+v, %v", c.state, rec, err) |
| + } |
| + } |
| + |
| + // Committed: the ordinary delete. |
| + f.seedRecord(t, "done.shop.example.com", "", newSeedID(t), seeds.KindImport, seeds.StateCommitted) |
| + if code, out := deleteInstance(t, f, "done.shop.example.com"); code != http.StatusOK || out["result"] != "deleted" { |
| + t.Fatalf("delete of a committed import = %d %v", code, out) |
| + } |
| + f.requireRecordGone(t, "done.shop.example.com") |
| +} |
| + |
| +// The pre-swap delete of a private-storage import tears down the empty |
| +// store its begin made (preflight m11). |
| +func TestImportDeleteTearsDownTheStore(t *testing.T) { |
| + f := newImportFixture(t) |
| + enableInstances(t, f.fx, fromApp, fromManifest(true)) |
| + code, out := beginImport(t, f, importHost, importSHA, 10) |
| + rec, _ := f.requireBegun(t, code, out, importHost) |
| + store, _, err := objstore.GetIn(context.Background(), f.fx.rel, "bab", fromApp, rec.Environment) |
| + if err != nil || store == nil || store.Declared == nil || store.Declared.Seed != rec.Seed.ID { |
| + t.Fatalf("store = %+v, %v; want one marked with seed %s", store, err, rec.Seed.ID) |
| + } |
| + bucket := storeBucketName(store) |
| + if _, ok := f.fake.Buckets[bucket]; !ok { |
| + t.Fatalf("bucket %s was never made", bucket) |
| + } |
| + if code, out := deleteInstance(t, f, importHost); code != http.StatusOK || out["result"] != "deleted" { |
| + t.Fatalf("delete = %d %v", code, out) |
| + } |
| + f.requireRecordGone(t, importHost) |
| + if store, _, err := objstore.GetIn(context.Background(), f.fx.rel, "bab", fromApp, rec.Environment); err != nil || store != nil { |
| + t.Fatalf("store after the delete = %+v, %v; want it gone", store, err) |
| + } |
| + if _, ok := f.fake.Buckets[bucket]; ok { |
| + t.Fatalf("bucket %s survived the delete", bucket) |
| + } |
| +} |
| + |
| +// failGetOn fails reads of key while armed. |
| +type failGetOn struct { |
| + release.Store |
| + key string |
| + armed atomic.Bool |
| +} |
| + |
| +func (s *failGetOn) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + if key == s.key && s.armed.Load() { |
| + return nil, "", errors.New("the bucket could not be read") |
| + } |
| + return s.Store.Get(ctx, key) |
| +} |
| + |
| +// The delete door's import rule fails closed (preflight M15, M8): a record |
| +// it cannot read, a finish that fails, and a seed id that cannot name a key |
| +// each answer main's delete sentence and delete nothing. A finish that |
| +// fails leaves the record rolled-back for a retry, which completes it. |
| +func TestImportDeleteFailsClosed(t *testing.T) { |
| + f := newImportFixture(t) |
| + id := newSeedID(t) |
| + f.seedRecord(t, importHost, "", id, seeds.KindImport, seeds.StateUploaded) |
| + |
| + rel := &failGetOn{Store: f.fx.rel, key: provision.RecordKey("bab", fromApp, importHost)} |
| + rel.armed.Store(true) |
| + f.fx.srv.cfg.Releases = rel |
| + if code, out := deleteInstance(t, f, importHost); code != http.StatusBadGateway || str(t, out, "error") != mainDeleteFailed { |
| + t.Fatalf("delete with an unreadable record = %d %v", code, out) |
| + } |
| + f.fx.srv.cfg.Releases = f.fx.rel |
| + rec, _ := f.record(t, importHost) |
| + requireImportState(t, rec, id, seeds.StateUploaded, "") |
| + |
| + data := f.fx.srv.cfg.SeedData |
| + f.fx.srv.cfg.SeedData = nil |
| + if code, out := deleteInstance(t, f, importHost); code != http.StatusBadGateway || str(t, out, "error") != mainDeleteFailed { |
| + t.Fatalf("delete with a failing finish = %d %v", code, out) |
| + } |
| + rec, _ = f.record(t, importHost) |
| + requireImportState(t, rec, id, seeds.StateRolledBack, "") |
| + if rows := importAudit(t, f, "instance-delete"); len(rows) != 0 { |
| + t.Fatalf("a failed delete audited: %q", rows) |
| + } |
| + f.fx.srv.cfg.SeedData = data |
| + if code, out := deleteInstance(t, f, importHost); code != http.StatusOK || out["result"] != "deleted" { |
| + t.Fatalf("retried delete = %d %v", code, out) |
| + } |
| + f.requireRecordGone(t, importHost) |
| + |
| + // A seed id that cannot name a key: nothing is written or deleted, and |
| + // the shared sentinel key survives. |
| + sentinel := seeds.SnapshotKey("bad.shop.example.com", "not-a-seed-id") |
| + if _, err := f.data.Put(context.Background(), sentinel, []byte("another record's object"), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.seedRecord(t, "bad.shop.example.com", "", "not-a-seed-id", seeds.KindImport, seeds.StateUploaded) |
| + if code, out := deleteInstance(t, f, "bad.shop.example.com"); code != http.StatusBadGateway || str(t, out, "error") != mainDeleteFailed { |
| + t.Fatalf("delete of a bad seed id = %d %v", code, out) |
| + } |
| + if rec, err := f.record(t, "bad.shop.example.com"); err != nil || rec.Seed.State != seeds.StateUploaded { |
| + t.Fatalf("record of a bad seed id = %+v, %v; want it untouched", rec, err) |
| + } |
| + if got := f.rawKey(t, f.data, sentinel); got != "another record's object" { |
| + t.Fatalf("sentinel = %q", got) |
| + } |
| +} |
| + |
| +// On a claim console the delete of an uncommitted import is refused as |
| +// every import door is: the finish runs nowhere there, and the claim's |
| +// ordinary retire would delete the record and leak what the seed made. |
| +func TestImportDeleteOnAClaimConsole(t *testing.T) { |
| + f := newImportFixture(t) |
| + id := newSeedID(t) |
| + f.seedRecord(t, importHost, "", id, seeds.KindImport, seeds.StateAwaitingUpload) |
| + f.fx.srv.cfg.HostClaim = true |
| + defer func() { f.fx.srv.cfg.HostClaim = false }() |
| + if code, out := deleteInstance(t, f, importHost); code != http.StatusConflict || str(t, out, "error") != seedClaimConsoleRefusal().Msg { |
| + t.Fatalf("delete on a claim console = %d %v", code, out) |
| + } |
| + rec, _ := f.record(t, importHost) |
| + requireImportState(t, rec, id, seeds.StateAwaitingUpload, "") |
| +} |
amadan