amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package runs
2
3import (
4 "bytes"
5 "context"
6 "crypto/rand"
7 "errors"
8 "regexp"
9 "strings"
10 "testing"
11 "testing/iotest"
12 "time"
13
14 "github.com/carlosframework/platform/internal/release"
15)
16
17func TestIDsAndKeys(t *testing.T) {
18 id, err := NewID(rand.Reader)
19 if err != nil || !regexp.MustCompile(`^run_[a-z2-7]{26}$`).MatchString(id) || !ValidID(id) {
20 t.Fatalf("id %q, %v", id, err)
21 }
22 for _, bad := range []string{"", "run_", "run_ABCDEFGHIJKLMNOPQRSTUVWXYZ", "run_abcdefghijklmnopqrstuvwxy/", "mv_abcdefghijklmnopqrstuvwxyz"} {
23 if ValidID(bad) {
24 t.Errorf("%q accepted", bad)
25 }
26 }
27 if got := Prefix("bab", "shop", id); got != "apps/bab/shop/runs/"+id+"/" {
28 t.Errorf("Prefix = %q", got)
29 }
30 k := ChunkKey("bab", "shop", id, 17, StreamErr)
31 if k != OutPrefix("bab", "shop", id)+"00000017-e" {
32 t.Errorf("ChunkKey = %q", k)
33 }
34 seq, stream, ok := ParseChunkKey(k, OutPrefix("bab", "shop", id))
35 if !ok || seq != 17 || stream != StreamErr {
36 t.Errorf("ParseChunkKey = %d %q %v", seq, stream, ok)
37 }
38 for _, bad := range []string{OutPrefix("bab", "shop", id) + "00000017-x", OutPrefix("bab", "shop", id) + "17-o", "elsewhere"} {
39 if _, _, ok := ParseChunkKey(bad, OutPrefix("bab", "shop", id)); ok {
40 t.Errorf("ParseChunkKey(%q) accepted", bad)
41 }
42 }
43}
44
45func TestCheckArgvAndTimeout(t *testing.T) {
46 cases := []struct {
47 argv []string
48 want error
49 }{
50 {nil, ErrNoArgs},
51 {make([]string, 257), ErrTooManyArgs},
52 {[]string{strings.Repeat("a", 64<<10+1)}, ErrArgvTooLarge},
53 {[]string{strings.Repeat("a", 64<<10-1), "b"}, nil}, // "at most 64 KB in total": the limit itself passes
54 {[]string{strings.Repeat("a", 32<<10), strings.Repeat("b", 32<<10+1)}, ErrArgvTooLarge},
55 {[]string{"a\x00b"}, ErrArgNUL},
56 {[]string{"import", "--dry-run"}, nil},
57 }
58 for i := range cases[1].argv {
59 cases[1].argv[i] = "x"
60 }
61 for _, c := range cases {
62 if err := CheckArgv(c.argv); !errors.Is(err, c.want) && !(c.want == nil && err == nil) {
63 t.Errorf("CheckArgv(%d args) = %v, want %v", len(c.argv), err, c.want)
64 }
65 }
66 for d, ok := range map[time.Duration]bool{0: false, 999 * time.Millisecond: false, time.Second: true, 6 * time.Hour: true, 6*time.Hour + time.Second: false} {
67 if (CheckTimeout(d) == nil) != ok {
68 t.Errorf("CheckTimeout(%s) = %v", d, CheckTimeout(d))
69 }
70 }
71}
72
73func TestCheckFiles(t *testing.T) {
74 if err := CheckFiles([]MetaFile{{Name: "a.csv", Size: 1}, {Name: "b.json", Size: 2}}); err != nil {
75 t.Fatal(err)
76 }
77 if err := CheckFiles([]MetaFile{{Name: "a", Size: MaxFileBytes}, {Name: "b", Size: 1}}); !errors.Is(err, ErrFilesTooLarge) {
78 t.Errorf("over the cap: %v", err)
79 }
80 for _, name := range []string{"", ".", "..", "a/b", "../x", "a\x00b", "a\nb", strings.Repeat("n", 256)} {
81 var fe *FileNameError
82 if err := CheckFiles([]MetaFile{{Name: name, Size: 1}}); !errors.Is(err, ErrFileName) || !errors.As(err, &fe) || fe.Name != name {
83 t.Errorf("name %q: %v", name, err)
84 }
85 }
86 if err := CheckFiles([]MetaFile{{Name: "a", Size: 1}, {Name: "a", Size: 1}}); !errors.Is(err, ErrFileTwice) {
87 t.Errorf("twice: %v", err)
88 }
89 if err := CheckFiles([]MetaFile{{Name: "a", Size: -1}}); !errors.Is(err, ErrFilesTooLarge) {
90 t.Errorf("a negative size must be refused: %v", err)
91 }
92}
93
94func TestStoreHelpers(t *testing.T) {
95 ctx := context.Background()
96 s := release.NewDirStore(t.TempDir())
97 id := "run_abcdefghijklmnopqrstuvwxyz"
98 if r, err := ReadResult(ctx, s, "bab", "shop", id); r != nil || err != nil {
99 t.Fatalf("absent result = %v, %v", r, err)
100 }
101 for seq := int64(1); seq <= 5; seq++ {
102 if _, err := s.Put(ctx, ChunkKey("bab", "shop", id, seq, StreamOut), []byte("x"), ""); err != nil {
103 t.Fatal(err)
104 }
105 }
106 keys, err := ListChunks(ctx, s, "bab", "shop", id, 2, 2)
107 if err != nil || len(keys) != 2 || !strings.HasSuffix(keys[0], "00000003-o") {
108 t.Fatalf("ListChunks = %q, %v", keys, err)
109 }
110 if last, err := LastChunkSeq(ctx, s, "bab", "shop", id); err != nil || last != 5 {
111 t.Fatalf("LastChunkSeq = %d, %v", last, err)
112 }
113 first := Result{Exit: 3, LastSeq: 5}
114 if err := WriteResult(ctx, s, "bab", "shop", id, first); err != nil {
115 t.Fatal(err)
116 }
117 if err := WriteResult(ctx, s, "bab", "shop", id, Result{Interrupted: InterruptedLost}); err != nil {
118 t.Fatalf("a second result is not an error: %v", err)
119 }
120 if r, err := ReadResult(ctx, s, "bab", "shop", id); err != nil || r.Exit != 3 || r.Interrupted != "" {
121 t.Fatalf("the first result must win: %+v, %v", r, err)
122 }
123 if err := DeleteRun(ctx, s, "bab", "shop", id); err != nil {
124 t.Fatal(err)
125 }
126 if left, _ := s.List(ctx, Prefix("bab", "shop", id)); len(left) != 0 {
127 t.Fatalf("DeleteRun left %q", left)
128 }
129}
130
131// A run id becomes one path element of an object key and of a directory on
132// the box, so ValidID must refuse anything that could be more or less.
133func TestValidIDIsOneCleanPathElement(t *testing.T) {
134 good := "run_abcdefghijklmnopqrstuvwxyz"
135 if !ValidID(good) {
136 t.Fatalf("%q refused", good)
137 }
138 for _, bad := range []string{
139 "", ".", "..", "run_", "/", "run_/", "../" + good, good + "/", good + "/..", good + "\n",
140 "\n" + good, good + "\x00", good + " ", "run_abcdefghijklmnopqrstuvwxy", good + "a",
141 "run_abcdefghijklmnopqrstuvwx\\z", "run_abcdefghijklmnopqrstuvwx.z", "run_abcdefghijklmnopqrstuvwxé",
142 "RUN_abcdefghijklmnopqrstuvwxyz", "run_abcdefghijklmnopqrstuvwx01",
143 } {
144 if ValidID(bad) {
145 t.Errorf("ValidID(%q) accepted", bad)
146 }
147 }
148}
149
150func TestNewIDDrawsFromTheReader(t *testing.T) {
151 // 16 bytes in, 128 bits of the id out: a different draw is a different id.
152 a, err := NewID(bytes.NewReader(bytes.Repeat([]byte{0x00}, 16)))
153 if err != nil || a != "run_"+strings.Repeat("a", 26) {
154 t.Fatalf("zero draw = %q, %v", a, err)
155 }
156 b, err := NewID(bytes.NewReader(append(bytes.Repeat([]byte{0x00}, 15), 0x01)))
157 if err != nil || b == a || !ValidID(b) {
158 t.Fatalf("last bit draw = %q, %v", b, err)
159 }
160 if _, err := NewID(bytes.NewReader(make([]byte, 15))); err == nil {
161 t.Error("a short read minted an id")
162 }
163 if _, err := NewID(iotest.ErrReader(errors.New("no entropy"))); err == nil {
164 t.Error("a failing reader minted an id")
165 }
166}
167
168func TestMintIDIsRandomAndValid(t *testing.T) {
169 seen := map[string]bool{}
170 for i := 0; i < 2000; i++ {
171 id, err := MintID()
172 if err != nil || !ValidID(id) {
173 t.Fatalf("MintID = %q, %v", id, err)
174 }
175 if seen[id] {
176 t.Fatalf("MintID repeated %q", id)
177 }
178 seen[id] = true
179 }
180}
181
182func TestParseChunkKeyIsCanonicalOnly(t *testing.T) {
183 out := OutPrefix("bab", "shop", "run_abcdefghijklmnopqrstuvwxyz")
184 for _, name := range []string{"+0000017-o", "-0000017-o", "0000000-o", "00000000-o", " 0000017-o", "0x000017-o", "0000017 -o"} {
185 if _, _, ok := ParseChunkKey(out+name, out); ok {
186 t.Errorf("ParseChunkKey(%q) accepted", name)
187 }
188 }
189}
190
191func TestStoreHelpersRefuseAnIdThatIsNotOne(t *testing.T) {
192 ctx := context.Background()
193 s := release.NewDirStore(t.TempDir())
194 if _, err := s.Put(ctx, "apps/bab/shop/runs/keep", []byte("x"), ""); err != nil {
195 t.Fatal(err)
196 }
197 for _, id := range []string{"", "..", "../x", "run_x/../..", "run_abcdefghijklmnopqrstuvwxyz/"} {
198 if _, err := ReadRequest(ctx, s, "bab", "shop", id); !errors.Is(err, ErrBadID) {
199 t.Errorf("ReadRequest(%q) = %v", id, err)
200 }
201 if _, err := ReadResult(ctx, s, "bab", "shop", id); !errors.Is(err, ErrBadID) {
202 t.Errorf("ReadResult(%q) = %v", id, err)
203 }
204 if err := WriteResult(ctx, s, "bab", "shop", id, Result{}); !errors.Is(err, ErrBadID) {
205 t.Errorf("WriteResult(%q) = %v", id, err)
206 }
207 if _, err := ListChunks(ctx, s, "bab", "shop", id, 0, 0); !errors.Is(err, ErrBadID) {
208 t.Errorf("ListChunks(%q) = %v", id, err)
209 }
210 if _, err := LastChunkSeq(ctx, s, "bab", "shop", id); !errors.Is(err, ErrBadID) {
211 t.Errorf("LastChunkSeq(%q) = %v", id, err)
212 }
213 if err := DeleteRun(ctx, s, "bab", "shop", id); !errors.Is(err, ErrBadID) {
214 t.Errorf("DeleteRun(%q) = %v", id, err)
215 }
216 }
217 if left, _ := s.List(ctx, "apps/bab/shop/runs/"); len(left) != 1 {
218 t.Fatalf("a refused delete touched the store: %q", left)
219 }
220}
221
222// acct and app are concatenated into every key, so a slash or ".." in either
223// would leave the app's subtree. Nothing may be read or written for them.
224func TestStoreHelpersRefuseAnAccountOrAppThatIsNotOne(t *testing.T) {
225 ctx := context.Background()
226 dir := t.TempDir()
227 s := release.NewDirStore(dir)
228 id := "run_abcdefghijklmnopqrstuvwxyz"
229 victim := RequestKey("bab", "x", id)
230 if _, err := s.Put(ctx, victim, []byte(`{}`), ""); err != nil {
231 t.Fatal(err)
232 }
233 before, _ := s.List(ctx, "")
234 for _, c := range [][2]string{{"bab/shop", "x"}, {"bab", "../x"}, {"bab", "../../y"}, {"", "shop"}, {"bab", ""}, {"bab", "a/b"}, {"..", "x"}, {"BAB", "x"}, {"bab", "x\n"}} {
235 acct, app := c[0], c[1]
236 if _, err := ReadRequest(ctx, s, acct, app, id); !errors.Is(err, ErrBadApp) {
237 t.Errorf("ReadRequest(%q, %q) = %v", acct, app, err)
238 }
239 if _, err := ReadResult(ctx, s, acct, app, id); !errors.Is(err, ErrBadApp) {
240 t.Errorf("ReadResult(%q, %q) = %v", acct, app, err)
241 }
242 if err := WriteResult(ctx, s, acct, app, id, Result{}); !errors.Is(err, ErrBadApp) {
243 t.Errorf("WriteResult(%q, %q) = %v", acct, app, err)
244 }
245 if _, err := ListChunks(ctx, s, acct, app, id, 0, 0); !errors.Is(err, ErrBadApp) {
246 t.Errorf("ListChunks(%q, %q) = %v", acct, app, err)
247 }
248 if _, err := LastChunkSeq(ctx, s, acct, app, id); !errors.Is(err, ErrBadApp) {
249 t.Errorf("LastChunkSeq(%q, %q) = %v", acct, app, err)
250 }
251 if err := DeleteRun(ctx, s, acct, app, id); !errors.Is(err, ErrBadApp) {
252 t.Errorf("DeleteRun(%q, %q) = %v", acct, app, err)
253 }
254 }
255 if after, _ := s.List(ctx, ""); strings.Join(after, "\n") != strings.Join(before, "\n") {
256 t.Fatalf("a refused call changed the store: %q -> %q", before, after)
257 }
258}
259
260// DeleteRun removes the marker last and request.json just before it, so a
261// run with any object left still has the marker that keeps its row single.
262func TestDeleteRunKeepsTheMarkerAndRequestUntilTheRestIsGone(t *testing.T) {
263 ctx := context.Background()
264 inner := release.NewDirStore(t.TempDir())
265 id, _ := MintID()
266 for _, k := range []string{AuditedKey("bab", "shop", id), RequestKey("bab", "shop", id), ResultKey("bab", "shop", id), ChunkKey("bab", "shop", id, 1, StreamOut)} {
267 if _, err := inner.Put(ctx, k, []byte("x"), ""); err != nil {
268 t.Fatal(err)
269 }
270 }
271 s := &failDelete{Store: inner, fail: ResultKey("bab", "shop", id)}
272 if err := DeleteRun(ctx, s, "bab", "shop", id); err == nil {
273 t.Fatal("a failed delete must be returned")
274 }
275 left, _ := inner.List(ctx, Prefix("bab", "shop", id))
276 if len(left) != 3 { // result, request, audited
277 t.Fatalf("left %q, want result.json, request.json and audited", left)
278 }
279 s.fail = ""
280 if err := DeleteRun(ctx, s, "bab", "shop", id); err != nil {
281 t.Fatal(err)
282 }
283 if left, _ := inner.List(ctx, Prefix("bab", "shop", id)); len(left) != 0 {
284 t.Fatalf("left %q", left)
285 }
286}
287
288// A chunk that will not delete stops DeleteRun before result.json, so a
289// follower of a half-deleted run still reads how it ended. Then the result
290// goes before request.json.
291func TestDeleteRunKeepsTheResultUntilTheOutputIsGone(t *testing.T) {
292 ctx := context.Background()
293 inner := release.NewDirStore(t.TempDir())
294 id, _ := MintID()
295 chunk := ChunkKey("bab", "shop", id, 1, StreamOut)
296 for _, k := range []string{AuditedKey("bab", "shop", id), RequestKey("bab", "shop", id), ResultKey("bab", "shop", id), chunk, InputKey("bab", "shop", id, "data.csv")} {
297 if _, err := inner.Put(ctx, k, []byte("{}"), ""); err != nil {
298 t.Fatal(err)
299 }
300 }
301 s := &failDelete{Store: inner, fail: chunk}
302 if err := DeleteRun(ctx, s, "bab", "shop", id); err == nil {
303 t.Fatal("a failed delete must be returned")
304 }
305 if r, err := ReadResult(ctx, inner, "bab", "shop", id); r == nil || err != nil {
306 t.Fatalf("result.json went before the output: %v, %v", r, err)
307 }
308 s.fail = RequestKey("bab", "shop", id)
309 if err := DeleteRun(ctx, s, "bab", "shop", id); err == nil {
310 t.Fatal("a failed delete must be returned")
311 }
312 if r, _ := ReadResult(ctx, inner, "bab", "shop", id); r != nil {
313 t.Fatal("result.json must go before request.json")
314 }
315}
316
317type failDelete struct {
318 release.Store
319 fail string
320}
321
322func (f *failDelete) Delete(ctx context.Context, key string) error {
323 if key == f.fail {
324 return errors.New("delete refused")
325 }
326 return f.Store.Delete(ctx, key)
327}
328
329func TestReleaseTimeoutMaxIsTheRunMax(t *testing.T) {
330 if got := time.Duration(release.MaxReleaseTimeoutSec) * time.Second; got != MaxTimeout {
331 t.Fatalf("release.MaxReleaseTimeoutSec = %v, runs.MaxTimeout = %v: spec §5.2.1 caps release at the carlos run maximum", got, MaxTimeout)
332 }
333}
334