| new file mode 100644 |
| index 000000000..c10be46e5 |
| --- /dev/null |
| +++ b/instances_import_test.go |
| @@ -0,0 +1,733 @@ |
| +package main |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "io" |
| + "net/http" |
| + "net/http/httptest" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "sync" |
| + "sync/atomic" |
| + "testing" |
| + "time" |
| +) |
| + |
| +func TestImportVerifyRetriesThroughTheWake(t *testing.T) { |
| + var calls atomic.Int32 |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + if r.Header.Get("Carlos-Import-Verify") != "tok" { |
| + w.WriteHeader(http.StatusForbidden) |
| + return |
| + } |
| + if calls.Add(1) < 3 { |
| + w.Header().Set("Retry-After", "0") |
| + w.WriteHeader(http.StatusServiceUnavailable) |
| + return |
| + } |
| + w.WriteHeader(http.StatusNoContent) |
| + })) |
| + defer ts.Close() |
| + code, err := verifyImportURL(context.Background(), ts.Client(), ts.URL+"/up", "tok") |
| + if err != nil || code != http.StatusNoContent || calls.Load() != 3 { |
| + t.Fatalf("code %d err %v calls %d", code, err, calls.Load()) |
| + } |
| +} |
| + |
| +func TestImportVerifyStopsOnAFailure(t *testing.T) { |
| + var calls atomic.Int32 |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + calls.Add(1) |
| + w.WriteHeader(http.StatusInternalServerError) |
| + })) |
| + defer ts.Close() |
| + code, err := verifyImportURL(context.Background(), ts.Client(), ts.URL+"/up", "tok") |
| + if err != nil || code != http.StatusInternalServerError || calls.Load() != 1 { |
| + t.Fatalf("code %d err %v calls %d", code, err, calls.Load()) |
| + } |
| +} |
| + |
| +// fakeSeedWait replaces the poll seam for one test: no real sleeps, and |
| +// every wait is recorded. |
| +func fakeSeedWait(t *testing.T, every time.Duration) *[]time.Duration { |
| + t.Helper() |
| + var mu sync.Mutex |
| + waits := &[]time.Duration{} |
| + oldWait, oldEvery := seedWait, seedPollEvery |
| + seedWait = func(ctx context.Context, d time.Duration) error { |
| + mu.Lock() |
| + *waits = append(*waits, d) |
| + mu.Unlock() |
| + return ctx.Err() |
| + } |
| + seedPollEvery = every |
| + t.Cleanup(func() { seedWait, seedPollEvery = oldWait, oldEvery }) |
| + return waits |
| +} |
| + |
| +// A gateway answer (502, 504) while the instance wakes is retried as a 503 |
| +// is, honouring Retry-After (M16). |
| +func TestImportVerifyRetriesGatewayAnswers(t *testing.T) { |
| + waits := fakeSeedWait(t, 2*time.Second) |
| + var calls atomic.Int32 |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + switch calls.Add(1) { |
| + case 1: |
| + w.WriteHeader(http.StatusBadGateway) |
| + case 2: |
| + w.Header().Set("Retry-After", "7") |
| + w.WriteHeader(http.StatusGatewayTimeout) |
| + default: |
| + w.WriteHeader(http.StatusOK) |
| + } |
| + })) |
| + defer ts.Close() |
| + code, err := verifyImportURL(context.Background(), ts.Client(), ts.URL+"/up", "tok") |
| + if err != nil || code != http.StatusOK || calls.Load() != 3 { |
| + t.Fatalf("code %d err %v calls %d", code, err, calls.Load()) |
| + } |
| + if len(*waits) != 2 || (*waits)[0] != 3*time.Second || (*waits)[1] != 7*time.Second { |
| + t.Fatalf("waits = %v, want 3s (no Retry-After) then 7s", *waits) |
| + } |
| +} |
| + |
| +func TestImportVerifyGivesUpAfterItsTries(t *testing.T) { |
| + fakeSeedWait(t, 2*time.Second) |
| + var calls atomic.Int32 |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + calls.Add(1) |
| + w.WriteHeader(http.StatusServiceUnavailable) |
| + })) |
| + defer ts.Close() |
| + code, err := verifyImportURL(context.Background(), ts.Client(), ts.URL+"/up", "tok") |
| + if err != nil || code != http.StatusServiceUnavailable || calls.Load() != importVerifyTries+1 { |
| + t.Fatalf("code %d err %v calls %d", code, err, calls.Load()) |
| + } |
| +} |
| + |
| +// A redirect is the verify's answer, never followed: /up answering 302 to a |
| +// login page that says 200 must not commit (M16). |
| +func TestImportVerifyDoesNotFollowARedirect(t *testing.T) { |
| + var landed atomic.Bool |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + if r.URL.Path == "/login" { |
| + landed.Store(true) |
| + w.WriteHeader(http.StatusOK) |
| + return |
| + } |
| + http.Redirect(w, r, "/login", http.StatusFound) |
| + })) |
| + defer ts.Close() |
| + client := ts.Client() // follows redirects by default |
| + code, err := verifyImportURL(context.Background(), client, ts.URL+"/up", "tok") |
| + if err != nil || code != http.StatusFound || landed.Load() { |
| + t.Fatalf("code %d err %v followed %v", code, err, landed.Load()) |
| + } |
| + if client.CheckRedirect != nil { |
| + t.Fatal("verifyImportURL changed the caller's client") |
| + } |
| +} |
| + |
| +func TestImportVerifyBuildsTheHostURL(t *testing.T) { |
| + var path, tok string |
| + ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + path, tok = r.URL.Path, r.Header.Get("Carlos-Import-Verify") |
| + w.WriteHeader(http.StatusOK) |
| + })) |
| + defer ts.Close() |
| + host := strings.TrimPrefix(ts.URL, "https://") |
| + code, err := verifyImport(context.Background(), ts.Client(), host, "up", "tok") |
| + if err != nil || code != http.StatusOK || path != "/up" || tok != "tok" { |
| + t.Fatalf("code %d err %v path %q token %q", code, err, path, tok) |
| + } |
| +} |
| + |
| +func TestImportClientCallsTheDoor(t *testing.T) { |
| + var seen []string |
| + var begin map[string]any |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + seen = append(seen, r.Method+" "+r.URL.Path) |
| + switch { |
| + case r.Method == http.MethodPost && r.URL.Path == "/api/cli/apps/bab/shop/instances": |
| + raw, _ := io.ReadAll(r.Body) |
| + _ = json.Unmarshal(raw, &begin) |
| + w.WriteHeader(http.StatusCreated) |
| + w.Write([]byte(`{"host":"old.shop.example.com","verify_token":"tok","seed":{"id":"sd_imp","kind":"import","state":"awaiting-upload"}}`)) |
| + default: |
| + w.WriteHeader(http.StatusOK) |
| + w.Write([]byte(`{"state":"ok"}`)) |
| + } |
| + })) |
| + defer ts.Close() |
| + s := newTestSession(ts.URL) |
| + a := apiAcct{Seg: "bab"} |
| + rec, tok, err := s.importBegin(context.Background(), a, "shop", "old.shop.example.com", "staging", "aa", 3, "127.0.0.1:8401", "moving the shop") |
| + if err != nil || tok != "tok" || rec.Seed == nil || rec.Seed.ID != "sd_imp" { |
| + t.Fatalf("begin: %q %+v %v", tok, rec, err) |
| + } |
| + imp, _ := begin["import"].(map[string]any) |
| + if begin["host"] != "old.shop.example.com" || begin["environment"] != "staging" || begin["message"] != "moving the shop" || |
| + imp["sha256"] != "aa" || imp["bytes"] != float64(3) || imp["replaces"] != "127.0.0.1:8401" { |
| + t.Fatalf("begin body = %v", begin) |
| + } |
| + for _, verb := range []string{"stopped", "commit", "abort"} { |
| + if err := s.importVerb(context.Background(), a, "shop", "old.shop.example.com", verb); err != nil { |
| + t.Fatalf("%s: %v", verb, err) |
| + } |
| + } |
| + want := []string{"POST /api/cli/apps/bab/shop/instances", "POST /api/cli/apps/bab/shop/instances/old.shop.example.com/import/stopped", |
| + "POST /api/cli/apps/bab/shop/instances/old.shop.example.com/import/commit", "POST /api/cli/apps/bab/shop/instances/old.shop.example.com/import/abort"} |
| + if len(seen) != len(want) { |
| + t.Fatalf("calls = %q", seen) |
| + } |
| + for i := range want { |
| + if seen[i] != want[i] { |
| + t.Fatalf("call %d = %q, want %q", i, seen[i], want[i]) |
| + } |
| + } |
| +} |
| + |
| +func TestImportBeginRefusalAndUnreadableAnswer(t *testing.T) { |
| + for _, c := range []struct { |
| + status int |
| + body string |
| + wantErr string |
| + }{ |
| + {http.StatusBadRequest, `{"error":"the sha256 is not 64 hex characters"}`, "the sha256 is not 64 hex characters"}, |
| + {http.StatusCreated, `{"host":"old.shop.example.com"}`, "instances import: unreadable response"}, |
| + {http.StatusCreated, `not json`, "instances import: unreadable response"}, |
| + } { |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + w.WriteHeader(c.status) |
| + io.WriteString(w, c.body) |
| + })) |
| + _, tok, err := newTestSession(ts.URL).importBegin(context.Background(), apiAcct{Seg: "bab"}, "shop", "old.shop.example.com", "", "aa", 3, "127.0.0.1:8401") |
| + ts.Close() |
| + if err == nil || !strings.HasPrefix(err.Error(), c.wantErr) || tok != "" { |
| + t.Errorf("%d %s: token %q err %v", c.status, c.body, tok, err) |
| + } |
| + } |
| +} |
| + |
| +// Begin can provision a private store (up to about 75 s), so it must not |
| +// ride the session's 30 s JSON client (M4). The session's client is shrunk |
| +// below the server's delay to show begin does not use it, and is left as |
| +// it was. |
| +func TestImportBeginOutlastsTheJSONClient(t *testing.T) { |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + time.Sleep(150 * time.Millisecond) |
| + w.WriteHeader(http.StatusCreated) |
| + io.WriteString(w, `{"host":"old.shop.example.com","verify_token":"tok"}`) |
| + })) |
| + defer ts.Close() |
| + s := newTestSession(ts.URL) |
| + s.c.httpc.Timeout = 20 * time.Millisecond |
| + if _, tok, err := s.importBegin(context.Background(), apiAcct{Seg: "bab"}, "shop", "old.shop.example.com", "", "aa", 3, "127.0.0.1:8401"); err != nil || tok != "tok" { |
| + t.Fatalf("token %q err %v", tok, err) |
| + } |
| + if s.c.httpc.Timeout != 20*time.Millisecond { |
| + t.Fatalf("the session's client timeout became %v", s.c.httpc.Timeout) |
| + } |
| +} |
| + |
| +func snapshotFile(t *testing.T, body string) *os.File { |
| + t.Helper() |
| + p := filepath.Join(t.TempDir(), "main.db") |
| + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + f, err := os.Open(p) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { f.Close() }) |
| + return f |
| +} |
| + |
| +func TestImportUploadStreamsTheFile(t *testing.T) { |
| + var got struct { |
| + method, path, expect, ctype string |
| + length int64 |
| + body string |
| + } |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + raw, _ := io.ReadAll(r.Body) |
| + got.method, got.path, got.expect, got.ctype = r.Method, r.URL.Path, r.Header.Get("Expect"), r.Header.Get("Content-Type") |
| + got.length, got.body = r.ContentLength, string(raw) |
| + time.Sleep(150 * time.Millisecond) |
| + w.WriteHeader(http.StatusAccepted) |
| + io.WriteString(w, `{"state":"checking"}`) |
| + })) |
| + defer ts.Close() |
| + s := newTestSession(ts.URL) |
| + s.c.httpc.Timeout = 20 * time.Millisecond // the upload never rides the JSON client |
| + if err := s.importUpload(context.Background(), apiAcct{Seg: "bab"}, "shop", "old.shop.example.com", snapshotFile(t, "abc"), 3); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got.method != http.MethodPut || got.path != "/api/cli/apps/bab/shop/instances/old.shop.example.com/import" || |
| + got.expect != "100-continue" || got.ctype != "application/octet-stream" || got.length != 3 || got.body != "abc" { |
| + t.Fatalf("request = %+v", got) |
| + } |
| +} |
| + |
| +func TestImportUploadRefusal(t *testing.T) { |
| + const sdN51 = "the upload does not match the length and sha256 this import began with, so it was not kept." |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + io.Copy(io.Discard, r.Body) |
| + w.WriteHeader(http.StatusUnprocessableEntity) |
| + fmt.Fprintf(w, `{"error":%q}`, sdN51) |
| + })) |
| + defer ts.Close() |
| + err := newTestSession(ts.URL).importUpload(context.Background(), apiAcct{Seg: "bab"}, "shop", "old.shop.example.com", snapshotFile(t, "abc"), 3) |
| + if err == nil || err.Error() != sdN51 { |
| + t.Fatalf("err = %v", err) |
| + } |
| +} |
| + |
| +// importConsole is a console for one import: begin, upload and the verbs |
| +// answer as configured, and the record walks through gets, one per look |
| +// (the last one repeats). |
| +type importConsole struct { |
| + mu sync.Mutex |
| + calls []string |
| + beginStatus int |
| + beginBody string |
| + upload func(w http.ResponseWriter) |
| + verbs map[string]func(w http.ResponseWriter) |
| + gets []func(w http.ResponseWriter) |
| + looks int |
| +} |
| + |
| +const importBeginBody = `{"host":"main.shop.example.com","verify_token":"s3cret-token","seed":{"id":"sd_imp","kind":"import","state":"awaiting-upload"}}` |
| + |
| +func newImportConsole(gets ...func(w http.ResponseWriter)) *importConsole { |
| + return &importConsole{beginStatus: http.StatusCreated, beginBody: importBeginBody, gets: gets, verbs: map[string]func(http.ResponseWriter){}} |
| +} |
| + |
| +func (c *importConsole) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| + c.mu.Lock() |
| + defer c.mu.Unlock() |
| + io.Copy(io.Discard, r.Body) |
| + path := r.URL.Path |
| + if i := strings.Index(path, "/instances"); i >= 0 { |
| + path = path[i:] |
| + } |
| + switch { |
| + case r.Method == http.MethodPost && path == "/instances": |
| + c.calls = append(c.calls, "begin") |
| + w.WriteHeader(c.beginStatus) |
| + io.WriteString(w, c.beginBody) |
| + case r.Method == http.MethodPut: |
| + c.calls = append(c.calls, "upload") |
| + if c.upload != nil { |
| + c.upload(w) |
| + return |
| + } |
| + w.WriteHeader(http.StatusAccepted) |
| + io.WriteString(w, `{"state":"checking"}`) |
| + case r.Method == http.MethodPost: |
| + verb := path[strings.LastIndex(path, "/")+1:] |
| + c.calls = append(c.calls, verb) |
| + if f := c.verbs[verb]; f != nil { |
| + f(w) |
| + return |
| + } |
| + fmt.Fprintf(w, `{"state":%q}`, verb) |
| + case r.Method == http.MethodGet: |
| + c.calls = append(c.calls, "get") |
| + i := c.looks |
| + if c.looks++; i >= len(c.gets) { |
| + i = len(c.gets) - 1 |
| + } |
| + c.gets[i](w) |
| + } |
| +} |
| + |
| +func (c *importConsole) called() []string { |
| + c.mu.Lock() |
| + defer c.mu.Unlock() |
| + return append([]string(nil), c.calls...) |
| +} |
| + |
| +// called reports whether the console saw verb. |
| +func (c *importConsole) saw(verb string) bool { |
| + for _, v := range c.called() { |
| + if v == verb { |
| + return true |
| + } |
| + } |
| + return false |
| +} |
| + |
| +func importRecord(state, reason string) func(w http.ResponseWriter) { |
| + return importRecordID("sd_imp", state, reason) |
| +} |
| + |
| +func importRecordID(id, state, reason string) func(w http.ResponseWriter) { |
| + return func(w http.ResponseWriter) { |
| + fmt.Fprintf(w, `{"host":"main.shop.example.com","seed":{"id":%q,"kind":"import","sha256":"ab","bytes":3,"state":%q,"reason":%q}}`, id, state, reason) |
| + } |
| +} |
| + |
| +// importRun runs an import of a three-byte file against c, with verify |
| +// (when not empty) answered by verifyStatus through a TLS server standing |
| +// in for the instance's host. It returns the output and the error. |
| +func importRun(t *testing.T, c *importConsole, verify string, verifyStatus ...int) (string, error) { |
| + t.Helper() |
| + ts := httptest.NewServer(c) |
| + defer ts.Close() |
| + j := importJob{acct: apiAcct{Seg: "bab"}, app: "shop", host: "main.shop.example.com", file: filepath.Join(t.TempDir(), "main.db"), |
| + replaces: "127.0.0.1:8401", verify: verify} |
| + if err := os.WriteFile(j.file, []byte("abc"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + if verify != "" { |
| + var n atomic.Int32 |
| + vs := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + if r.Header.Get("Carlos-Import-Verify") != "s3cret-token" { |
| + w.WriteHeader(http.StatusForbidden) |
| + return |
| + } |
| + i := int(n.Add(1)) - 1 |
| + if i >= len(verifyStatus) { |
| + i = len(verifyStatus) - 1 |
| + } |
| + w.WriteHeader(verifyStatus[i]) |
| + })) |
| + defer vs.Close() |
| + j.host, j.verifyClient = strings.TrimPrefix(vs.URL, "https://"), vs.Client() |
| + } |
| + var out bytes.Buffer |
| + err := newTestSession(ts.URL).runImport(context.Background(), &out, j) |
| + // The output names the file and the host as the tests know them. |
| + return strings.NewReplacer(j.file, "main.db", j.host, "main.shop.example.com").Replace(out.String()), err |
| +} |
| + |
| +func wantLines(t *testing.T, out string, want ...string) { |
| + t.Helper() |
| + got := strings.Split(strings.TrimSuffix(out, "\n"), "\n") |
| + if len(got) != len(want) { |
| + t.Fatalf("output:\n%s\nwant:\n%s", out, strings.Join(want, "\n")) |
| + } |
| + for i := range want { |
| + if got[i] != want[i] { |
| + t.Fatalf("line %d = %q, want %q\noutput:\n%s", i, got[i], want[i], out) |
| + } |
| + } |
| +} |
| + |
| +const ( |
| + importCommitLine = "commit with: carlos instances import commit --app shop --host main.shop.example.com. Abort with: carlos instances import abort --app shop --host main.shop.example.com." |
| + importTokenLine = "verify token: s3cret-token. It is shown only this once." |
| +) |
| + |
| +func TestImportWithoutVerifyPrintsTheTokenAndTheCommands(t *testing.T) { |
| + waits := fakeSeedWait(t, 2*time.Second) |
| + c := newImportConsole(importRecord("awaiting-upload", ""), importRecord("uploaded", ""), |
| + importRecord("ready", ""), importRecord("swapped", "")) |
| + out, err := importRun(t, c, "") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + wantLines(t, out, |
| + "uploading main.db (3B)", |
| + importTokenLine, |
| + "uploaded. The old process of main.shop.example.com is marked stopped, and the box swaps it on its next pass.", |
| + "waiting for the box to swap main.shop.example.com", |
| + "swapped. Until it is committed, main.shop.example.com answers only requests that carry the verify token.", |
| + importCommitLine) |
| + if got := strings.Join(c.called(), " "); got != "begin upload get get stopped get get" { |
| + t.Fatalf("calls = %s", got) |
| + } |
| + for _, d := range *waits { |
| + if d != 2*time.Second { |
| + t.Fatalf("waits = %v", *waits) |
| + } |
| + } |
| +} |
| + |
| +func TestImportWithVerifyCommits(t *testing.T) { |
| + fakeSeedWait(t, 2*time.Second) |
| + c := newImportConsole(importRecord("uploaded", ""), importRecord("swapped", ""), importRecord("swapped", ""), importRecord("placed", "")) |
| + out, err := importRun(t, c, "/up", http.StatusServiceUnavailable, http.StatusNoContent) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if strings.Contains(out, "s3cret-token") { |
| + t.Fatalf("a verified import never shows the token:\n%s", out) |
| + } |
| + lines := strings.Split(strings.TrimSuffix(out, "\n"), "\n") |
| + wantLines(t, strings.Join(lines[len(lines)-3:], "\n"), |
| + "verify passed (204). Committing.", |
| + "waiting for the box to finish placing the snapshot", |
| + "committed. main.shop.example.com is served by the platform now.") |
| + if !c.saw("commit") || c.saw("abort") { |
| + t.Fatalf("calls = %v", c.called()) |
| + } |
| +} |
| + |
| +func TestImportWithVerifyAbortsOnAFailure(t *testing.T) { |
| + fakeSeedWait(t, 2*time.Second) |
| + c := newImportConsole(importRecord("uploaded", ""), importRecord("swapped", "")) |
| + out, err := importRun(t, c, "/up", http.StatusInternalServerError) |
| + if !errors.Is(err, errImportFailed) { |
| + t.Fatalf("err = %v", err) |
| + } |
| + lines := strings.Split(strings.TrimSuffix(out, "\n"), "\n") |
| + wantLines(t, strings.Join(lines[len(lines)-2:], "\n"), |
| + "verify failed (500). Aborting the import.", |
| + "aborted. The box puts the old route of main.shop.example.com back on its next pass.") |
| + if strings.Contains(out, "s3cret-token") || !c.saw("abort") || c.saw("commit") { |
| + t.Fatalf("calls = %v, output:\n%s", c.called(), out) |
| + } |
| +} |
| + |
| +func TestImportRefusedByTheBox(t *testing.T) { |
| + fakeSeedWait(t, 2*time.Second) |
| + c := newImportConsole(importRecord("uploaded", ""), importRecord("ready", ""), importRecord("refused", "the route belongs to app kass")) |
| + out, err := importRun(t, c, "/up", http.StatusOK) |
| + if !errors.Is(err, errImportFailed) { |
| + t.Fatalf("err = %v", err) |
| + } |
| + if !strings.HasSuffix(out, "waiting for the box to swap "+hostOf(out)+"\nthe box refused the import: the route belongs to app kass\n") { |
| + t.Fatalf("output:\n%s", out) |
| + } |
| + if strings.Contains(out, "s3cret-token") || c.saw("commit") || c.saw("abort") { |
| + t.Fatalf("calls = %v, output:\n%s", c.called(), out) |
| + } |
| +} |
| + |
| +// hostOf reads the host back out of the "waiting for the box to swap" line, |
| +// for the tests whose host is a TLS server's address. |
| +func hostOf(out string) string { |
| + const p = "waiting for the box to swap " |
| + i := strings.Index(out, p) |
| + if i < 0 { |
| + return "" |
| + } |
| + rest := out[i+len(p):] |
| + return rest[:strings.Index(rest, "\n")] |
| +} |
| + |
| +// The swap wait is ten minutes of looks; at its end the token is shown |
| +// (with --verify) and SD-N79 names the way on (M17). |
| +func TestImportSwapDeadline(t *testing.T) { |
| + waits := fakeSeedWait(t, time.Minute) |
| + c := newImportConsole(importRecord("uploaded", ""), importRecord("ready", "")) |
| + out, err := importRun(t, c, "/up", http.StatusOK) |
| + if !errors.Is(err, errImportFailed) { |
| + t.Fatalf("err = %v", err) |
| + } |
| + host := hostOf(out) |
| + if !strings.HasSuffix(out, "waiting for the box to swap "+host+"\n"+importTokenLine+"\n"+ |
| + "the box has not swapped "+host+" after 10 minutes. Check carlos instances list, then commit or abort.\n") { |
| + t.Fatalf("output:\n%s", out) |
| + } |
| + // One wait before the upload's look, then ten minutes of one-minute waits. |
| + if len(*waits) != 11 { |
| + t.Fatalf("waits = %v", *waits) |
| + } |
| +} |
| + |
| +func TestImportPlaceDeadline(t *testing.T) { |
| + fakeSeedWait(t, time.Minute) |
| + c := newImportConsole(importRecord("uploaded", ""), importRecord("swapped", "")) |
| + out, err := importRun(t, c, "/up", http.StatusOK) |
| + if !errors.Is(err, errImportFailed) { |
| + t.Fatalf("err = %v", err) |
| + } |
| + host := hostOf(out) |
| + if !strings.HasSuffix(out, "waiting for the box to finish placing the snapshot\n"+importTokenLine+"\n"+ |
| + "commit with: carlos instances import commit --app shop --host "+host+". Abort with: carlos instances import abort --app shop --host "+host+".\n") { |
| + t.Fatalf("output:\n%s", out) |
| + } |
| + if c.saw("commit") || c.saw("abort") { |
| + t.Fatalf("calls = %v", c.called()) |
| + } |
| +} |
| + |
| +func TestImportAbortedPrintsItsReason(t *testing.T) { |
| + const sdN52 = "the upload is not a database that passes an integrity check, so it was not kept." |
| + for _, c := range []struct { |
| + name, reason, want string |
| + }{ |
| + {"with a reason", sdN52, sdN52}, |
| + {"without one", "", "aborting"}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + fakeSeedWait(t, 2*time.Second) |
| + ic := newImportConsole(importRecord("awaiting-upload", ""), importRecord("aborted", c.reason)) |
| + out, err := importRun(t, ic, "/up", http.StatusOK) |
| + if !errors.Is(err, errImportFailed) { |
| + t.Fatalf("err = %v", err) |
| + } |
| + if !strings.HasSuffix(out, "\n"+c.want+"\n") || strings.Contains(out, "s3cret-token") || ic.saw("stopped") { |
| + t.Fatalf("calls = %v, output:\n%s", ic.called(), out) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestImportRolledBackWhileWaitingForTheSwap(t *testing.T) { |
| + fakeSeedWait(t, 2*time.Second) |
| + c := newImportConsole(importRecord("uploaded", ""), importRecord("ready", ""), importRecord("rolled-back", "")) |
| + out, err := importRun(t, c, "") |
| + if !errors.Is(err, errImportFailed) || !strings.HasSuffix(out, "waiting for the box to swap main.shop.example.com\naborting\n") { |
| + t.Fatalf("err = %v, output:\n%s", err, out) |
| + } |
| +} |
| + |
| +func TestImportAnotherImportOnTheHostEndsTheWait(t *testing.T) { |
| + fakeSeedWait(t, 2*time.Second) |
| + c := newImportConsole(importRecord("uploaded", ""), importRecordID("sd_other", "ready", "")) |
| + out, err := importRun(t, c, "") |
| + if !errors.Is(err, errImportFailed) || !strings.HasSuffix(out, "\naborting\n") { |
| + t.Fatalf("err = %v, output:\n%s", err, out) |
| + } |
| +} |
| + |
| +// A stopped whose first answer was lost answers 409 SD-N54 naming a state |
| +// past it; the record says the stop landed, so the import goes on. |
| +func TestImportStoppedThatAlreadyLanded(t *testing.T) { |
| + fakeSeedWait(t, 2*time.Second) |
| + c := newImportConsole(importRecord("uploaded", ""), importRecord("ready", ""), importRecord("swapped", "")) |
| + c.verbs["stopped"] = answerWith(http.StatusConflict, `{"error":"main.shop.example.com is ready. Mark it stopped once, after its upload."}`) |
| + out, err := importRun(t, c, "") |
| + if err != nil || !strings.HasSuffix(out, importCommitLine+"\n") { |
| + t.Fatalf("err = %v, output:\n%s", err, out) |
| + } |
| +} |
| + |
| +func TestImportStoppedRefusedForReal(t *testing.T) { |
| + fakeSeedWait(t, 2*time.Second) |
| + const refusal = "carlos run is busy on main.shop.example.com. Wait for it to finish, then abort." |
| + c := newImportConsole(importRecord("uploaded", "")) |
| + c.verbs["stopped"] = answerWith(http.StatusConflict, fmt.Sprintf(`{"error":%q}`, refusal)) |
| + out, err := importRun(t, c, "/up", http.StatusOK) |
| + if err == nil || err.Error() != refusal { |
| + t.Fatalf("err = %v", err) |
| + } |
| + // With --verify, an exit that leaves the import live shows the token (M17). |
| + if !strings.HasSuffix(out, importTokenLine+"\n") { |
| + t.Fatalf("output:\n%s", out) |
| + } |
| +} |
| + |
| +func TestImportUploadRefusedShowsTheTokenWithVerify(t *testing.T) { |
| + fakeSeedWait(t, 2*time.Second) |
| + const sdN51 = "the upload does not match the length and sha256 this import began with, so it was not kept." |
| + c := newImportConsole(importRecord("awaiting-upload", "")) |
| + c.upload = answerWith(http.StatusUnprocessableEntity, fmt.Sprintf(`{"error":%q}`, sdN51)) |
| + out, err := importRun(t, c, "/up", http.StatusOK) |
| + if err == nil || err.Error() != sdN51 || strings.Contains(err.Error(), "s3cret-token") { |
| + t.Fatalf("err = %v", err) |
| + } |
| + if !strings.HasSuffix(out, importTokenLine+"\n") || c.saw("get") { |
| + t.Fatalf("calls = %v, output:\n%s", c.called(), out) |
| + } |
| +} |
| + |
| +func TestImportBeginRefusedShowsNothingMore(t *testing.T) { |
| + fakeSeedWait(t, 2*time.Second) |
| + c := newImportConsole() |
| + c.beginStatus, c.beginBody = http.StatusConflict, `{"error":"instance main.shop.example.com already exists"}` |
| + out, err := importRun(t, c, "") |
| + if err == nil || err.Error() != "instance main.shop.example.com already exists" { |
| + t.Fatalf("err = %v", err) |
| + } |
| + wantLines(t, out, "uploading main.db (3B)") |
| + if got := c.called(); len(got) != 1 { |
| + t.Fatalf("calls = %v", got) |
| + } |
| +} |
| + |
| +// A commit that lost its answer is settled by the record (an overlapping |
| +// commit answers SD-N60; the record is committed). |
| +func TestImportCommitSettledByTheRecord(t *testing.T) { |
| + fakeSeedWait(t, 2*time.Second) |
| + c := newImportConsole(importRecord("uploaded", ""), importRecord("swapped", ""), importRecord("placed", ""), importRecord("committed", "")) |
| + c.verbs["commit"] = answerWith(http.StatusConflict, `{"error":"main.shop.example.com changed while this request was made. Read its state with carlos instances list and try again."}`) |
| + out, err := importRun(t, c, "/up", http.StatusOK) |
| + if err != nil || !strings.HasSuffix(out, " is served by the platform now.\n") { |
| + t.Fatalf("err = %v, output:\n%s", err, out) |
| + } |
| +} |
| + |
| +// An abort that answers 404 after the import's own record has gone is done. |
| +func TestImportAbortOfAFinishedImport(t *testing.T) { |
| + fakeSeedWait(t, 2*time.Second) |
| + c := newImportConsole(importRecord("uploaded", ""), importRecord("swapped", ""), answerWith(http.StatusNotFound, `{"error":"x is not an instance of shop"}`)) |
| + c.verbs["abort"] = answerWith(http.StatusNotFound, `{"error":"x is not an instance of shop"}`) |
| + out, err := importRun(t, c, "/up", http.StatusBadGateway, http.StatusForbidden) |
| + if !errors.Is(err, errImportFailed) || !strings.HasSuffix(out, "Aborting the import.\naborted. The box puts the old route of "+hostOf(out)+" back on its next pass.\n") { |
| + t.Fatalf("err = %v, output:\n%s", err, out) |
| + } |
| +} |
| + |
| +func TestImportVerbAlone(t *testing.T) { |
| + for _, c := range []struct { |
| + name, verb string |
| + answer func(http.ResponseWriter) |
| + record func(http.ResponseWriter) |
| + want string |
| + wantErr string |
| + }{ |
| + {"commit", "commit", nil, nil, "committed. main.shop.example.com is served by the platform now.", ""}, |
| + {"abort", "abort", nil, nil, "aborted. The box puts the old route of main.shop.example.com back on its next pass.", ""}, |
| + {"commit refused", "commit", answerWith(http.StatusConflict, `{"error":"main.shop.example.com is swapped. Commit it once it is placed."}`), |
| + importRecord("swapped", ""), "", "main.shop.example.com is swapped. Commit it once it is placed."}, |
| + {"abort that lost to its own retry", "abort", answerWith(http.StatusConflict, `{"error":"changed"}`), |
| + importRecord("aborted", ""), "aborted. The box puts the old route of main.shop.example.com back on its next pass.", ""}, |
| + {"abort of a host with no record", "abort", answerWith(http.StatusNotFound, `{"error":"main.shop.example.com is not an instance of shop"}`), |
| + answerWith(http.StatusNotFound, ""), "", "main.shop.example.com is not an instance of shop"}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + ic := newImportConsole(c.record) |
| + if c.answer != nil { |
| + ic.verbs[c.verb] = c.answer |
| + } |
| + ts := httptest.NewServer(ic) |
| + defer ts.Close() |
| + var out bytes.Buffer |
| + err := newTestSession(ts.URL).runImportVerb(context.Background(), &out, apiAcct{Seg: "bab"}, "shop", "main.shop.example.com", c.verb) |
| + if c.wantErr != "" { |
| + if err == nil || err.Error() != c.wantErr || out.Len() != 0 { |
| + t.Fatalf("err = %v, output %q", err, out.String()) |
| + } |
| + return |
| + } |
| + if err != nil || out.String() != c.want+"\n" { |
| + t.Fatalf("err = %v, output %q", err, out.String()) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// The shared poller (M18): a bounded wait ends with DeadlineExceeded after |
| +// its share of looks; failed looks count as Plan 5's do. |
| +func TestPollSeedLimitAndFailures(t *testing.T) { |
| + waits := fakeSeedWait(t, 2*time.Second) |
| + c := newImportConsole(importRecord("ready", "")) |
| + ts := httptest.NewServer(c) |
| + defer ts.Close() |
| + s := newTestSession(ts.URL) |
| + _, err := s.pollSeed(context.Background(), apiAcct{Seg: "bab"}, "shop", "main.shop.example.com", 10*time.Second, importLook("sd_imp", "swapped")) |
| + if !errors.Is(err, context.DeadlineExceeded) || len(*waits) != 5 { |
| + t.Fatalf("err = %v, waits = %v", err, *waits) |
| + } |
| + bad := newImportConsole(answerWith(http.StatusBadGateway, `{"error":"could not read the instance"}`)) |
| + ts2 := httptest.NewServer(bad) |
| + defer ts2.Close() |
| + _, err = newTestSession(ts2.URL).pollSeed(context.Background(), apiAcct{Seg: "bab"}, "shop", "main.shop.example.com", 0, importLook("sd_imp", "swapped")) |
| + bad.mu.Lock() |
| + looks := bad.looks |
| + bad.mu.Unlock() |
| + if err == nil || err.Error() != "could not read the instance" || looks != seedPollFailures { |
| + t.Fatalf("err = %v, looks = %d", err, looks) |
| + } |
| +} |
amadan