carlos / platform Public

Download this file

1	package release
2
3	import (
4	"context"
5	"fmt"
6	"net/http"
7	"net/http/httptest"
8	"testing"
9	)
10
11	func TestDirStorePageListerOrdersAndBounds(t *testing.T) {
12	ctx := context.Background()
13	d := NewDirStore(t.TempDir())
14	for _, k := range []string{"q/c", "q/a", "q/e", "q/b", "q/d", "other/z"} {
15	if _, err := d.Put(ctx, k, []byte("x"), ""); err != nil {
16	t.Fatal(err)
17	}
18	}
19
20	// Sorted, bounded, and the prefix is respected.
21	got, err := d.ListPage(ctx, "q/", "", 3)
22	if err != nil {
23	t.Fatal(err)
24	}
25	want := []string{"q/a", "q/b", "q/c"}
26	if fmt.Sprint(got) != fmt.Sprint(want) {
27	t.Fatalf("first page = %v, want %v", got, want)
28	}
29
30	// startAfter is EXCLUSIVE, so paging is "the last key I saw".
31	got, err = d.ListPage(ctx, "q/", "q/c", 3)
32	if err != nil {
33	t.Fatal(err)
34	}
35	if want := []string{"q/d", "q/e"}; fmt.Sprint(got) != fmt.Sprint(want) {
36	t.Fatalf("second page = %v, want %v", got, want)
37	}
38
39	// Paging to the end terminates rather than repeating.
40	got, err = d.ListPage(ctx, "q/", "q/e", 3)
41	if err != nil || len(got) != 0 {
42	t.Fatalf("past the end = %v, %v", got, err)
43	}
44
45	// limit <= 0 means no bound.
46	got, err = d.ListPage(ctx, "q/", "", 0)
47	if err != nil || len(got) != 5 {
48	t.Fatalf("unbounded = %v, %v", got, err)
49	}
50	}
51
52	// The S3 half, against an in-process fake rather than a live bucket.
53	//
54	// The point of this test is that ListPage puts start-after and max-keys ON THE
55	// WIRE. Nothing else in the package does — the other listings paginate through
56	// everything — and a drain that silently enumerated a flooded app's whole
57	// queue every 30 seconds would still look correct from the outside.
58	func TestS3StorePageListerBoundsTheRequest(t *testing.T) {
59	var gotStartAfter, gotMaxKeys, gotPrefix string
60	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
61	q := r.URL.Query()
62	gotStartAfter, gotMaxKeys, gotPrefix = q.Get("start-after"), q.Get("max-keys"), q.Get("prefix")
63	w.Header().Set("Content-Type", "application/xml")
64	fmt.Fprint(w, `<?xml version="1.0" encoding="UTF-8"?>
65	<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
66	<Name>b</Name><KeyCount>2</KeyCount><IsTruncated>true</IsTruncated>
67	<Contents><Key>q/d</Key><Size>1</Size></Contents>
68	<Contents><Key>q/e</Key><Size>1</Size></Contents>
69	</ListBucketResult>`)
70	}))
71	defer srv.Close()
72
73	rgwEnv(t, srv.URL)
74	s, err := NewS3Store(context.Background(), "b")
75	if err != nil {
76	t.Fatal(err)
77	}
78
79	keys, err := s.ListPage(context.Background(), "q/", "q/c", 2)
80	if err != nil {
81	t.Fatal(err)
82	}
83	if fmt.Sprint(keys) != fmt.Sprint([]string{"q/d", "q/e"}) {
84	t.Fatalf("keys = %v", keys)
85	}
86	if gotStartAfter != "q/c" {
87	t.Fatalf("start-after = %q, want the caller's cursor on the wire", gotStartAfter)
88	}
89	if gotMaxKeys != "2" {
90	t.Fatalf("max-keys = %q, want the bound on the wire — otherwise the cap is a lie", gotMaxKeys)
91	}
92	if gotPrefix != "q/" {
93	t.Fatalf("prefix = %q", gotPrefix)
94	}
95	}
96
97	// An empty cursor sends no start-after at all, rather than an empty one.
98	func TestS3StorePageListerOmitsEmptyCursor(t *testing.T) {
99	var sawStartAfter bool
100	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
101	_, sawStartAfter = r.URL.Query()["start-after"]
102	w.Header().Set("Content-Type", "application/xml")
103	fmt.Fprint(w, `<?xml version="1.0" encoding="UTF-8"?>
104	<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>b</Name><KeyCount>0</KeyCount></ListBucketResult>`)
105	}))
106	defer srv.Close()
107
108	rgwEnv(t, srv.URL)
109	s, err := NewS3Store(context.Background(), "b")
110	if err != nil {
111	t.Fatal(err)
112	}
113	if _, err := s.ListPage(context.Background(), "q/", "", 10); err != nil {
114	t.Fatal(err)
115	}
116	if sawStartAfter {
117	t.Fatal("an empty cursor was sent as start-after=")
118	}
119	}
120
121	// "limit <= 0 means no bound" has to be true of BOTH stores, or a caller
122	// asking for everything gets everything from DirStore and the first thousand
123	// keys from S3 — with nothing to say there was more. One ListObjectsV2 is
124	// capped at a thousand and reports IsTruncated; the unbounded path has to
125	// follow it.
126	func TestS3StorePageListerUnboundedFollowsTruncation(t *testing.T) {
127	var pages int
128	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
129	pages++
130	w.Header().Set("Content-Type", "application/xml")
131	if pages == 1 {
132	// A truncated first page, exactly as S3 answers one.
133	fmt.Fprint(w, `<?xml version="1.0" encoding="UTF-8"?>
134	<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
135	<Name>b</Name><KeyCount>2</KeyCount><IsTruncated>true</IsTruncated>
136	<NextContinuationToken>tok</NextContinuationToken>
137	<Contents><Key>q/a</Key><Size>1</Size></Contents>
138	<Contents><Key>q/b</Key><Size>1</Size></Contents>
139	</ListBucketResult>`)
140	return
141	}
142	if got := r.URL.Query().Get("continuation-token"); got != "tok" {
143	t.Errorf("second request carried continuation-token %q, want the token the first page gave", got)
144	}
145	fmt.Fprint(w, `<?xml version="1.0" encoding="UTF-8"?>
146	<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
147	<Name>b</Name><KeyCount>1</KeyCount><IsTruncated>false</IsTruncated>
148	<Contents><Key>q/c</Key><Size>1</Size></Contents>
149	</ListBucketResult>`)
150	}))
151	defer srv.Close()
152
153	rgwEnv(t, srv.URL)
154	s, err := NewS3Store(context.Background(), "b")
155	if err != nil {
156	t.Fatal(err)
157	}
158	keys, err := s.ListPage(context.Background(), "q/", "", 0)
159	if err != nil {
160	t.Fatal(err)
161	}
162	if fmt.Sprint(keys) != fmt.Sprint([]string{"q/a", "q/b", "q/c"}) {
163	t.Fatalf("unbounded listing = %v, want every key across both pages", keys)
164	}
165	if pages != 2 {
166	t.Fatalf("made %d requests, want the truncation followed", pages)
167	}
168	}
169
170	// And the BOUNDED case stays one request, which is the whole reason this
171	// interface exists: a drain that paged through a flooded app's entire queue
172	// every tick would be paying for the listing it deliberately declined to act
173	// on.
174	func TestS3StorePageListerBoundedStaysOneRequest(t *testing.T) {
175	var requests int
176	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
177	requests++
178	w.Header().Set("Content-Type", "application/xml")
179	fmt.Fprint(w, `<?xml version="1.0" encoding="UTF-8"?>
180	<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
181	<Name>b</Name><KeyCount>1</KeyCount><IsTruncated>true</IsTruncated>
182	<NextContinuationToken>tok</NextContinuationToken>
183	<Contents><Key>q/a</Key><Size>1</Size></Contents>
184	</ListBucketResult>`)
185	}))
186	defer srv.Close()
187
188	rgwEnv(t, srv.URL)
189	s, err := NewS3Store(context.Background(), "b")
190	if err != nil {
191	t.Fatal(err)
192	}
193	if _, err := s.ListPage(context.Background(), "q/", "", 1); err != nil {
194	t.Fatal(err)
195	}
196	if requests != 1 {
197	t.Fatalf("a bounded listing made %d requests, want exactly one", requests)
198	}
199	}
200