amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux
2
3package host
4
5import (
6 "bytes"
7 "errors"
8 "fmt"
9 "os"
10 "path/filepath"
11 "testing"
12
13 "github.com/carlosframework/platform/internal/provision"
14 "golang.org/x/sys/unix"
15)
16
17func displacedParkFixture(t *testing.T, oldRecord bool) (*servingParkDiskFixture, servingParkRecord, servingParkQuarantine, string) {
18 t.Helper()
19 f := newServingParkDiskFixture(t)
20 f.put("", "last tenant writes")
21 f.put("-wal", "last WAL")
22 f.put("-shm", "last SHM")
23 a, stops := f.evidence(10, true, "missing")
24 if _, err := f.park(a, stops); err != nil {
25 t.Fatal(err)
26 }
27 f.complete(a)
28 r, err := f.ledger.read(a.ID)
29 if err != nil {
30 t.Fatal(err)
31 }
32 if oldRecord {
33 r.Namespace = servingArtifactDirectory{}
34 if err := f.ledger.save(r); err != nil {
35 t.Fatal(err)
36 }
37 }
38 path := filepath.Join(f.data, "instances", ".serving-parked")
39 saved := path + "-saved"
40 if err := os.Rename(path, saved); err != nil {
41 t.Fatal(err)
42 }
43 if err := os.Mkdir(path, 0700); err != nil {
44 t.Fatal(err)
45 }
46 if err := os.WriteFile(filepath.Join(path, "unknown"), []byte("keep these bytes"), 0600); err != nil {
47 t.Fatal(err)
48 }
49 q, err := quarantineRecover(t, f, r.Source.Parent)
50 if err != nil || q.Previous == nil {
51 t.Fatal("quarantine", err)
52 }
53 return f, r, q, saved
54}
55
56func TestServingDisplacedParkReplacementAndExpiry(t *testing.T) {
57 for _, old := range []bool{false, true} {
58 for _, replacement := range []bool{false, true} {
59 t.Run(fmt.Sprintf("old=%v/replacement=%v", old, replacement), func(t *testing.T) {
60 f, r, q, saved := displacedParkFixture(t, old)
61 before, err := f.disk.Read("retirement-" + r.Authorization.ID + ".json")
62 if err != nil {
63 t.Fatal(err)
64 }
65 if err := f.fs.CheckParked(&r); err != nil {
66 t.Fatal("known displaced park inaccessible", err)
67 }
68 after, err := f.disk.Read("retirement-" + r.Authorization.ID + ".json")
69 if err != nil || !bytes.Equal(before, after) {
70 t.Fatal("inspection wrote authority", err)
71 }
72 f.put("", "current tenant writes")
73 if replacement {
74 next, stops := f.evidence(11, false, "started")
75 if _, err := f.park(next, stops); err != nil {
76 t.Fatal("replacement", err)
77 }
78 f.complete(next)
79 body, err := os.ReadFile(f.path(next, ""))
80 if err != nil || string(body) != "current tenant writes" {
81 t.Fatal("replacement database", err)
82 }
83 } else {
84 f.now = r.ReapAfter
85 if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), r.Authorization.ID) }); err != nil {
86 t.Fatal("expiry", err)
87 }
88 body, err := os.ReadFile(provision.NestedDBPath(f.data, f.base.Host))
89 if err != nil || string(body) != "current tenant writes" {
90 t.Fatal("expiry touched live database", err)
91 }
92 }
93 got, err := f.ledger.read(r.Authorization.ID)
94 if err != nil || !got.Closed || !got.EventRemoved || got.Namespace != q.Previous.Root || !got.ReapAfter.Equal(r.ReapAfter) {
95 t.Fatal("closed record lost identity or deadline", err)
96 }
97 if _, err := os.Stat(filepath.Join(saved, r.Authorization.ID)); !errors.Is(err, os.ErrNotExist) {
98 t.Fatal("old event remains", err)
99 }
100 if err := f.ledger.recoverCompleted(t.Context(), r.Authorization.ID); err != nil {
101 t.Fatal("closed replay", err)
102 }
103 quarantinePreserved(t, f, q)
104 })
105 }
106 }
107}
108
109func TestServingDisplacedParkRetainsUnprovenIdentities(t *testing.T) {
110 for _, fault := range []string{"parent", "parent-mode", "root", "root-mode", "symlink", "nested", "event", "extra-child", "hardlink", "file", "quarantine", "limit"} {
111 t.Run(fault, func(t *testing.T) {
112 f, r, q, saved := displacedParkFixture(t, false)
113 event := filepath.Join(saved, r.Authorization.ID)
114 db := filepath.Join(event, r.Authorization.Successor.Host+".db")
115 keep := db
116 switch fault {
117 case "parent":
118 parent := filepath.Dir(saved)
119 if err := os.Rename(parent, parent+"-saved"); err != nil {
120 t.Fatal(err)
121 }
122 if err := os.Mkdir(parent, 0700); err != nil {
123 t.Fatal(err)
124 }
125 keep = filepath.Join(parent+"-saved", filepath.Base(saved), r.Authorization.ID, filepath.Base(db))
126 case "parent-mode":
127 if err := os.Chmod(filepath.Dir(saved), 0777); err != nil {
128 t.Fatal(err)
129 }
130 case "root", "symlink", "nested":
131 outside := filepath.Join(f.data, "retained")
132 if fault == "nested" {
133 outside = filepath.Join(filepath.Dir(saved), "container", "retained")
134 if err := os.Mkdir(filepath.Dir(outside), 0700); err != nil {
135 t.Fatal(err)
136 }
137 }
138 if err := os.Rename(saved, outside); err != nil {
139 t.Fatal(err)
140 }
141 keep = filepath.Join(outside, r.Authorization.ID, filepath.Base(db))
142 if fault == "symlink" {
143 if err := os.Symlink(outside, saved); err != nil {
144 t.Fatal(err)
145 }
146 } else if err := os.Mkdir(saved, 0700); err != nil {
147 t.Fatal(err)
148 }
149 case "root-mode":
150 if err := os.Chmod(saved, 0755); err != nil {
151 t.Fatal(err)
152 }
153 case "event":
154 if err := os.Rename(event, event+"-saved"); err != nil {
155 t.Fatal(err)
156 }
157 if err := os.Mkdir(event, 0700); err != nil {
158 t.Fatal(err)
159 }
160 keep = filepath.Join(event+"-saved", filepath.Base(db))
161 case "extra-child":
162 if err := os.WriteFile(filepath.Join(event, "unknown"), []byte("unrecorded"), 0600); err != nil {
163 t.Fatal(err)
164 }
165 case "hardlink":
166 if err := os.Link(db, filepath.Join(f.data, "other-link")); err != nil {
167 t.Fatal(err)
168 }
169 case "file":
170 if err := os.Rename(db, db+"-saved"); err != nil {
171 t.Fatal(err)
172 }
173 if err := os.WriteFile(db, []byte("victim"), 0600); err != nil {
174 t.Fatal(err)
175 }
176 keep = db + "-saved"
177 case "quarantine":
178 if err := f.disk.Write("namespace-quarantine.json", []byte("{}")); err != nil {
179 t.Fatal(err)
180 }
181 case "limit":
182 entries, err := os.ReadDir(filepath.Dir(saved))
183 if err != nil {
184 t.Fatal(err)
185 }
186 // Exactly one over the bound: the limited read includes the
187 // genuine root regardless of the filesystem's enumeration order.
188 for i := len(entries); i < servingParkRootSearchLimit+1; i++ {
189 if err := os.Mkdir(filepath.Join(filepath.Dir(saved), fmt.Sprintf("unknown-%d", i)), 0000); err != nil {
190 t.Fatal(err)
191 }
192 }
193 }
194 before, err := f.disk.Read("retirement-" + r.Authorization.ID + ".json")
195 if err != nil {
196 t.Fatal(err)
197 }
198 f.now = r.ReapAfter
199 if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), r.Authorization.ID) }); err == nil {
200 t.Fatal("unproven identity deleted")
201 }
202 got, err := os.ReadFile(keep)
203 if err != nil || string(got) != "last tenant writes" {
204 t.Fatal("retained bytes changed", err)
205 }
206 // Deletion intent may be recorded, but no file progress is permitted.
207 after, err := f.ledger.read(r.Authorization.ID)
208 if err != nil {
209 t.Fatal(err)
210 }
211 for _, file := range after.Source.Files {
212 if file.DeleteIntent || file.Deleted {
213 t.Fatal("unsafe deletion progress")
214 }
215 }
216 if after.Closed || after.EventRemoveIntent || len(before) == 0 {
217 t.Fatal("unsafe closure")
218 }
219 if fault != "parent" {
220 quarantinePreserved(t, f, q)
221 }
222 })
223 }
224}
225
226func TestServingDisplacedParkRecoveryWriteBoundaries(t *testing.T) {
227 for _, fault := range []string{"root-before", "root-after", "unlink", "rmdir"} {
228 t.Run(fault, func(t *testing.T) {
229 f, r, q, saved := displacedParkFixture(t, true)
230 f.now = r.ReapAfter
231 fired := false
232 check := func(name string, body []byte) error {
233 var next servingParkRecord
234 if !fired && name == "retirement-"+r.Authorization.ID+".json" && servingParkDecode(body, &next) == nil && next.Namespace.Inode != 0 {
235 fired = true
236 return unix.ENOSPC
237 }
238 return nil
239 }
240 if fault == "root-before" {
241 f.ledger.state = quarantineWriteFault{Store: f.disk, before: check}
242 }
243 if fault == "root-after" {
244 f.ledger.state = quarantineWriteFault{Store: f.disk, after: check}
245 }
246 if fault == "unlink" || fault == "rmdir" {
247 f.fs.unlink = func(fd int, name string, flags int) error {
248 if err := unix.Unlinkat(fd, name, flags); err != nil {
249 return err
250 }
251 if !fired && (fault == "unlink" && flags == 0 || fault == "rmdir" && flags == unix.AT_REMOVEDIR) {
252 fired = true
253 return unix.ENOSPC
254 }
255 return nil
256 }
257 }
258 reap := func() error {
259 return f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), r.Authorization.ID) })
260 }
261 if err := reap(); !errors.Is(err, unix.ENOSPC) || !fired {
262 t.Fatal("missing boundary", err)
263 }
264 if fault == "root-before" || fault == "root-after" {
265 body, err := os.ReadFile(filepath.Join(saved, r.Authorization.ID, r.Authorization.Successor.Host+".db"))
266 if err != nil || string(body) != "last tenant writes" {
267 t.Fatal("deleted before durable root", err)
268 }
269 }
270 f.ledger.state = f.disk
271 fs := newServingParkFS(f.data, f.disk)
272 fs.owner, fs.mount = f.fs.owner, f.fs.mount
273 f.fs, f.ledger.fs = fs, fs
274 if err := f.ledger.recoverCompleted(t.Context(), r.Authorization.ID); err != nil {
275 t.Fatal("recovery", err)
276 }
277 got, err := f.ledger.read(r.Authorization.ID)
278 if err != nil || !got.Closed || !got.EventRemoved || got.Namespace != q.Previous.Root {
279 t.Fatal("incomplete recovered deletion", err)
280 }
281 quarantinePreserved(t, f, q)
282 })
283 }
284}
285
286func TestServingDisplacedParkCannotAdoptEventUnderUnknownRoot(t *testing.T) {
287 for _, old := range []bool{false, true} {
288 t.Run(fmt.Sprint(old), func(t *testing.T) {
289 f, r, q, saved := displacedParkFixture(t, old)
290 outside := filepath.Join(f.data, "known-root")
291 if err := os.Rename(saved, outside); err != nil {
292 t.Fatal(err)
293 }
294 if err := os.Mkdir(saved, 0700); err != nil {
295 t.Fatal(err)
296 }
297 if err := os.Rename(filepath.Join(outside, r.Authorization.ID), filepath.Join(saved, r.Authorization.ID)); err != nil {
298 t.Fatal(err)
299 }
300 f.now = r.ReapAfter
301 if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), r.Authorization.ID) }); err == nil {
302 t.Fatal("event identity authorized an unknown root")
303 }
304 body, err := os.ReadFile(filepath.Join(saved, r.Authorization.ID, r.Authorization.Successor.Host+".db"))
305 if err != nil || string(body) != "last tenant writes" {
306 t.Fatal("unknown root lost known event", err)
307 }
308 quarantinePreserved(t, f, q)
309 })
310 }
311}
312
313func TestServingDisplacedParkPendingRenameResumesOriginalSlot(t *testing.T) {
314 f := newServingParkDiskFixture(t)
315 f.put("", "interrupted database")
316 f.put("-wal", "interrupted WAL")
317 a, stops := f.evidence(10, true, "missing")
318 f.fs.rename = func(a int, b string, c int, d string, flags uint) error {
319 if err := unix.Renameat2(a, b, c, d, flags); err != nil {
320 return err
321 }
322 return unix.EIO
323 }
324 if _, err := f.park(a, stops); !errors.Is(err, unix.EIO) {
325 t.Fatal("missing interrupted rename", err)
326 }
327 r, err := f.ledger.read(a.ID)
328 if err != nil || r.Complete || r.Parked {
329 t.Fatal("pending record", err)
330 }
331 path := filepath.Join(f.data, "instances", ".serving-parked")
332 if err := os.Rename(path, path+"-saved"); err != nil {
333 t.Fatal(err)
334 }
335 if err := os.Mkdir(path, 0700); err != nil {
336 t.Fatal(err)
337 }
338 if err := os.WriteFile(filepath.Join(path, "unknown"), []byte("keep these bytes"), 0600); err != nil {
339 t.Fatal(err)
340 }
341 f.fs.rename = unix.Renameat2
342 q, err := quarantineRecover(t, f, r.Source.Parent)
343 if err != nil {
344 t.Fatal(err)
345 }
346 if _, err := f.park(a, stops); err != nil {
347 t.Fatal("pending replay", err)
348 }
349 f.complete(a)
350 slot, err := f.ledger.slot(a)
351 if err != nil || slot.ID != a.ID || slot.Pending != "" {
352 t.Fatal("recovery changed slot", err)
353 }
354 for suffix, want := range map[string]string{"": "interrupted database", "-wal": "interrupted WAL"} {
355 body, err := os.ReadFile(filepath.Join(path+"-saved", a.ID, a.Successor.Host+".db"+suffix))
356 if err != nil || string(body) != want {
357 t.Fatal("pending recovery lost bytes", err)
358 }
359 }
360 quarantinePreserved(t, f, q)
361}
362
363func TestServingDisplacedParkOldAbsentEventRemainsAmbiguous(t *testing.T) {
364 f, r, q, saved := displacedParkFixture(t, true)
365 // Model an older deletion whose rmdir reply was lost before the root was
366 // pinned. Neither namespace's absence supplies the missing identity.
367 r.Deleting, r.Cause, r.EventRemoveIntent = true, "expiry", true
368 for i := range r.Source.Files {
369 item := &r.Source.Files[i]
370 if item.Present {
371 item.DeleteIntent, item.Deleted = true, true
372 if err := os.Remove(filepath.Join(saved, r.Authorization.ID, item.Name)); err != nil {
373 t.Fatal(err)
374 }
375 }
376 }
377 if err := os.Remove(filepath.Join(saved, r.Authorization.ID)); err != nil {
378 t.Fatal(err)
379 }
380 if err := f.ledger.save(r); err != nil {
381 t.Fatal(err)
382 }
383 before, err := f.disk.Read("retirement-" + r.Authorization.ID + ".json")
384 if err != nil {
385 t.Fatal(err)
386 }
387 if err := f.ledger.recoverCompleted(t.Context(), r.Authorization.ID); err == nil {
388 t.Fatal("old absence accepted as root identity")
389 }
390 after, err := f.disk.Read("retirement-" + r.Authorization.ID + ".json")
391 if err != nil || !bytes.Equal(before, after) {
392 t.Fatal("ambiguous recovery wrote progress", err)
393 }
394 if _, err := os.Stat(filepath.Join(saved, r.Authorization.ID)); !errors.Is(err, os.ErrNotExist) {
395 t.Fatal("ambiguous event recreated", err)
396 }
397 quarantinePreserved(t, f, q)
398}
399