amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "bytes"
5 "context"
6 "crypto/ed25519"
7 "encoding/json"
8 "errors"
9 "log"
10 "os"
11 "path/filepath"
12 "strings"
13 "sync"
14 "testing"
15
16 "github.com/carlosframework/platform/internal/activator"
17 "github.com/carlosframework/platform/internal/appconfig"
18 "github.com/carlosframework/platform/internal/hostfloor"
19 "github.com/carlosframework/platform/internal/provision"
20 "github.com/carlosframework/platform/internal/registry"
21 "github.com/carlosframework/platform/internal/release"
22 "github.com/carlosframework/platform/internal/rootparticipation"
23 "github.com/carlosframework/platform/internal/rootstate"
24)
25
26// Model try-restart's no-op for a stopped service, and reset-failed's
27// transition to inactive. Restart represents the operator's subsequent start.
28type r2Units struct {
29 *plan14RestartUnits
30 showErr error
31 tryErr error
32 tries []string
33 starts []string
34}
35
36func (u *r2Units) Show(unit string) (unitProps, error) { return u.props, u.showErr }
37func (u *r2Units) TryRestart(unit string) error {
38 u.tries = append(u.tries, unit)
39 if u.tryErr != nil {
40 return u.tryErr
41 }
42 if u.props.ActiveState == "inactive" || u.props.ActiveState == "failed" {
43 return nil
44 }
45 return u.Restart(unit)
46}
47func (u *r2Units) ResetFailed(unit string) error {
48 u.effects = append(u.effects, "reset "+unit)
49 if u.props.MainPID == 0 {
50 u.props.ActiveState, u.props.SubState = "inactive", "dead"
51 }
52 return nil
53}
54
55func (u *r2Units) Start(unit string) error {
56 u.starts = append(u.starts, unit)
57 if u.props.ActiveState == "active" {
58 return nil
59 }
60 return u.Restart(unit)
61}
62
63type r2Live struct {
64 e *proofRootEnv
65 rt registry.Route
66 u *r2Units
67 c rootRestartChecks
68 source *proofBindingSource
69 pub ed25519.PublicKey
70 key ed25519.PrivateKey
71}
72
73func r2Fixture(t *testing.T, sourceKind string) *r2Live {
74 t.Helper()
75 e, rt, base, c, _ := plan14AdoptionFixture(t)
76 u := &r2Units{plan14RestartUnits: base}
77 if err := e.store.Delete(t.Context(), provision.RecordKey(rt.Account, rt.App, rt.Host)); err != nil {
78 t.Fatal(err)
79 }
80 rt.Account = ""
81 if err := e.root.reg.Upsert(rt); err != nil {
82 t.Fatal(err)
83 }
84 s := productionProofBindingSource(e.store, u, u.opt)
85 s.namedPath = filepath.Join(t.TempDir(), "named-units.json")
86 s.unitDirs = []string{filepath.Dir(u.props.FragmentPath)}
87 s.healthy = nil
88 put := func(path string, body []byte) {
89 t.Helper()
90 if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
91 t.Fatal(err)
92 }
93 if err := os.WriteFile(path, body, 0644); err != nil {
94 t.Fatal(err)
95 }
96 makeProofAncestorsPublic(t, path)
97 }
98 makeProofAncestorsPublic(t, s.namedPath)
99 makeProofAncestorsPublic(t, u.exe)
100 makeProofAncestorsPublic(t, u.props.FragmentPath)
101 switch sourceKind {
102 case "guard":
103 put(filepath.Join(u.opt, "system/guard/console-ready.json"), []byte(release.LiveLink(u.opt, rt.Host)))
104 drop := filepath.Join(s.unitDirs[0], "console.service.d/90-host-floor.conf")
105 put(drop, []byte(hostfloor.ConsoleGuardDropIn(release.LiveLink(u.opt, rt.Host))))
106 u.props.DropInPaths = []string{drop}
107 u.props.ExecPath = "/opt/carlos/system/guard/carlos"
108 put(u.props.ExecPath, []byte("guard"))
109 case "list":
110 body, err := json.Marshal(struct {
111 Schema int `json:"schema"`
112 Units []namedUnit `json:"units"`
113 }{1, []namedUnit{{Host: rt.Host, App: rt.App, Unit: rt.Unit, Addr: rt.Addr, Health: &provision.Health{Path: "/ready"}}}})
114 if err != nil {
115 t.Fatal(err)
116 }
117 put(s.namedPath, body)
118 }
119 c.resolve, c.host, c.rollback, c.prepareRollback, c.dispatch, c.targetFile = s.resolve, s.forHost, s.rollback, s.prepareRollback, s.needsRestart, rootstate.OpenTrustedFile
120 c.legacy, c.legacyLog, c.legacyReady = legacyRootRestart, &sync.Once{}, c.ready
121 pub, key, err := ed25519.GenerateKey(nil)
122 if err != nil {
123 t.Fatal(err)
124 }
125 if err := e.store.Delete(t.Context(), release.ChannelKey("bab", rt.App, rt.Channel)); err != nil {
126 t.Fatal(err)
127 }
128 f := &r2Live{e: e, rt: rt, u: u, c: c, source: s, pub: pub, key: key}
129 f.promote(t, "v1")
130 f.promote(t, "v2")
131 return f
132}
133func (f *r2Live) promote(t *testing.T, v string) {
134 t.Helper()
135 if err := release.Promote(t.Context(), f.e.store, f.key, "bab", f.rt.App, f.rt.Channel, v); err != nil {
136 t.Fatal(err)
137 }
138}
139func (f *r2Live) restart() func(string) error {
140 return rootRestartRouteWith(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, nil, f.c)
141}
142func (f *r2Live) pass(t *testing.T, extra ...release.PassOption) error {
143 start := f.c
144 start.legacyStart = f.u.Start
145 opts := []release.PassOption{release.WithRunningVersion(f.c.runningVersion), release.WithRestartFailurePropagation(), release.WithStartUnit(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, start)), release.WithRollbackRestart(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c))}
146 return release.Pass(t.Context(), f.e.root.reg, f.e.store, f.pub, f.u.opt, f.restart(), append(opts, extra...)...)
147}
148func (f *r2Live) wantLive(t *testing.T, v string) {
149 t.Helper()
150 got, err := release.LiveVersion(f.u.opt, f.rt.Host)
151 if err != nil || got != v {
152 t.Fatalf("live=%s, err=%v; want %s", got, err, v)
153 }
154}
155func (f *r2Live) wantRunning(t *testing.T, v string) {
156 t.Helper()
157 if f.u.props.MainPID <= 0 || f.u.exe != release.VersionPath(f.u.opt, "", v, f.rt.App) {
158 t.Fatalf("not running %s: %+v", v, f.u)
159 }
160}
161
162func TestR2UnguardedConsoleUpdate(t *testing.T) {
163 if !plan14RootFixture(t) {
164 return
165 }
166 for _, kind := range []string{"none", "list"} {
167 t.Run(kind, func(t *testing.T) {
168 f := r2Fixture(t, kind)
169 var logged bytes.Buffer
170 previousLog := log.Writer()
171 log.SetOutput(&logged)
172 defer log.SetOutput(previousLog)
173 if err := f.pass(t); err != nil {
174 t.Fatal(err)
175 }
176 f.wantLive(t, "v2")
177 f.wantRunning(t, "v2")
178 if len(f.u.effects) != 1 {
179 t.Fatal(f.u.effects)
180 }
181 // The same callback serves config sync and restart-only requests.
182 rt := f.rt
183 rt.Backing = registry.BackingUnit
184 if err := f.e.root.reg.Upsert(rt); err != nil {
185 t.Fatal(err)
186 }
187 if err := appconfig.SetEnv(t.Context(), f.e.store, "bab", rt.App, map[string]string{"LOG_LEVEL": "debug"}); err != nil {
188 t.Fatal(err)
189 }
190 dir := t.TempDir()
191 for i := 0; i < 2; i++ {
192 if err := appconfig.Sync(t.Context(), f.e.root.reg, f.e.store, nil, dir, f.restart(), nil); err != nil {
193 t.Fatal(err)
194 }
195 }
196 if len(f.u.effects) != 2 {
197 t.Fatal("config sync missed or repeated restart", f.u.effects)
198 }
199 routes, err := f.e.root.reg.List()
200 if err != nil {
201 t.Fatal(err)
202 }
203 marks := filepath.Join(t.TempDir(), "restart.json")
204 restartPass(t.Context(), routes, f.e.store, marks, f.restart())
205 askRestart(t, t.Context(), f.e.store, "bab", rt.App)
206 restartPass(t.Context(), routes, f.e.store, marks, f.restart())
207 restartPass(t.Context(), routes, f.e.store, marks, f.restart())
208 if len(f.u.effects) != 3 {
209 t.Fatal("restart-only missed or repeated restart", f.u.effects)
210 }
211 if kind == "none" && strings.Count(logged.String(), "legacy restart (no root census)") != 1 {
212 t.Fatal("legacy restart diagnostic was not once per pass", logged.String())
213 }
214 })
215 }
216}
217
218func TestR2UnguardedIncidentRecipe(t *testing.T) {
219 if !plan14RootFixture(t) {
220 return
221 }
222 for _, kind := range []string{"none", "guard", "list"} {
223 for _, order := range []string{"reset before adoption", "reset after adoption", "never failed"} {
224 t.Run(kind+"/"+order, func(t *testing.T) {
225 f := r2Fixture(t, kind)
226 if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil {
227 t.Fatal(err)
228 }
229 if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil {
230 t.Fatal(err)
231 }
232 if err := f.e.root.reg.SetVersions(f.rt.Host, "v2", "v1"); err != nil {
233 t.Fatal(err)
234 }
235 f.u.exe = release.VersionPath(f.u.opt, "", "v2", f.rt.App)
236 f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = "failed", "failed", 0
237 if order == "never failed" {
238 f.u.props.ActiveState, f.u.props.SubState = "inactive", "dead"
239 }
240 if order == "reset before adoption" {
241 if err := f.u.ResetFailed(f.rt.Unit); err != nil {
242 t.Fatal(err)
243 }
244 }
245 before := len(f.u.effects)
246 if kind != "none" {
247 if _, err := f.source.forHost(t.Context(), f.rt.Host); err != nil {
248 t.Fatal("PID-zero binding rejected", err)
249 }
250 }
251 f.promote(t, "v1")
252 if err := f.pass(t); err != nil {
253 t.Fatal(err)
254 }
255 f.wantLive(t, "v1")
256 if order != "reset after adoption" && (f.u.props.MainPID != 0 || len(f.u.effects) != before) {
257 t.Fatal("woke inactive unit", f.u.effects)
258 }
259 if order == "reset after adoption" {
260 if err := f.u.ResetFailed(f.rt.Unit); err != nil {
261 t.Fatal(err)
262 }
263 }
264 if err := f.u.Restart(f.rt.Unit); err != nil {
265 t.Fatal(err)
266 }
267 f.wantRunning(t, "v1")
268 })
269 }
270 }
271}
272
273func TestR2GuardedAutoRestartFixForward(t *testing.T) {
274 if !plan14RootFixture(t) {
275 return
276 }
277 for _, state := range []unitProps{{ActiveState: "activating", SubState: "auto-restart"}, {ActiveState: "activating", SubState: "start"}} {
278 t.Run(state.SubState, func(t *testing.T) {
279 f := r2Fixture(t, "guard")
280 f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = state.ActiveState, state.SubState, 0
281 if err := f.pass(t); err != nil {
282 t.Fatal(err)
283 }
284 f.wantLive(t, "v2")
285 f.wantRunning(t, "v2")
286 // Without a census, main try-restarts the activating unit once.
287 if len(f.u.effects) != 1 || len(f.u.tries) != 1 || f.u.tries[0] != "console.service" {
288 t.Fatal(f.u.effects, f.u.tries)
289 }
290 })
291 }
292}
293
294func TestR2BoxLocalUnitStamp(t *testing.T) {
295 if !plan14RootFixture(t) {
296 return
297 }
298 for _, recordUnit := range []string{"", "custom.service"} {
299 t.Run(recordUnit, func(t *testing.T) {
300 f := r2Fixture(t, "none")
301 // A socket-derived record cannot authorize a custom named unit. The
302 // no-census fallback still uses main's box-local stamp.
303 plan14PutRecord(t, f.e.store, provision.Record{Schema: 1, Host: f.rt.Host, Account: "bab", App: f.rt.App, Channel: f.rt.Channel, Unit: recordUnit})
304 f.rt.Unit = "titogo-canary@t.service"
305 f.u.props.Name = f.rt.Unit
306 if err := f.e.root.reg.Upsert(f.rt); err != nil {
307 t.Fatal(err)
308 }
309 if err := f.pass(t); err != nil {
310 t.Fatal(err)
311 }
312 f.wantRunning(t, "v2")
313 if len(f.u.tries) != 1 || f.u.tries[0] != f.rt.Unit {
314 t.Fatal(f.u.tries)
315 }
316 })
317 }
318}
319
320type r2ListFailure struct {
321 release.Store
322 lists int
323}
324
325func (s *r2ListFailure) List(context.Context, string) ([]string, error) {
326 s.lists++
327 return nil, os.ErrPermission
328}
329func TestR2OrdinaryRestartWithListFailure(t *testing.T) {
330 if !plan14RootFixture(t) {
331 return
332 }
333 for _, kind := range []string{"unit", "exec", "derived stamp"} {
334 t.Run(kind, func(t *testing.T) {
335 f := r2Fixture(t, "none")
336 st := &r2ListFailure{Store: f.e.store}
337 f.source.store = st
338 f.rt.Unit = ""
339 f.rt.Backing = registry.BackingUnit
340 if kind == "exec" {
341 f.rt.Backing = "exec"
342 }
343 if kind == "derived stamp" {
344 f.rt.Unit = activator.UnitName(f.rt.Host)
345 }
346 if err := f.e.root.reg.Delete(f.rt.Host); err != nil {
347 t.Fatal(err)
348 }
349 if err := f.e.root.reg.Upsert(f.rt); err != nil {
350 t.Fatal(err)
351 }
352 if err := f.pass(t); err != nil {
353 t.Fatal(err)
354 }
355 f.wantLive(t, "v2")
356 f.wantRunning(t, "v2")
357 if st.lists != 0 || len(f.u.tries) != 1 || f.u.tries[0] != activator.UnitName(f.rt.Host) {
358 t.Fatal(st.lists, f.u.tries)
359 }
360 })
361 }
362}
363
364func TestR2NamedRestartWithListFailure(t *testing.T) {
365 if !plan14RootFixture(t) {
366 return
367 }
368 f := r2Fixture(t, "none")
369 f.source.store = &r2ListFailure{Store: f.e.store}
370 if err := f.pass(t); err != nil {
371 t.Fatal(err)
372 }
373 f.wantLive(t, "v2")
374 f.wantRunning(t, "v2")
375}
376
377func TestR2OrdinaryRestartMatchesMain(t *testing.T) {
378 t.Parallel()
379 for _, scenario := range []string{"show error", "exec mismatch", "activating", "inactive", "missing row", "registry error", "try-restart error"} {
380 t.Run(scenario, func(t *testing.T) {
381 e, rt, base, c, _ := plan14AdoptionFixture(t)
382 u := &r2Units{plan14RestartUnits: base}
383 rt.Unit = ""
384 rt.Backing = registry.BackingUnit
385 if err := e.root.reg.Delete(rt.Host); err != nil {
386 t.Fatal(err)
387 }
388 if err := e.root.reg.Upsert(rt); err != nil {
389 t.Fatal(err)
390 }
391 c.legacy = func() (bool, error) { return true, nil }
392 c.dispatch = func(context.Context, registry.Route) (bool, error) {
393 t.Fatal("ordinary restart consulted records")
394 return false, nil
395 }
396 switch scenario {
397 case "show error":
398 u.showErr = os.ErrPermission
399 case "exec mismatch":
400 u.props.ExecPath = "/different"
401 case "activating":
402 u.props.ActiveState = "activating"
403 case "inactive":
404 u.props.ActiveState = "inactive"
405 case "missing row":
406 if err := e.root.reg.Delete(rt.Host); err != nil {
407 t.Fatal(err)
408 }
409 case "registry error":
410 e.root.reg.Close()
411 case "try-restart error":
412 u.tryErr = os.ErrPermission
413 }
414 if err := rootRestartRouteWith(e.root.reg, u, u.opt, nil, nil, c)(rt.Host); err != nil {
415 t.Fatal(err)
416 }
417 if len(u.tries) != 1 || u.tries[0] != activator.UnitName(rt.Host) {
418 t.Fatal(u.tries)
419 }
420 if scenario == "inactive" && len(u.effects) != 0 {
421 t.Fatal("woke inactive ordinary unit")
422 }
423 if err := rollbackUnitRestart(e.root.reg, u, u.opt, nil, c)(rt.Host); err != nil {
424 t.Fatal("ordinary rollback reports outage", err)
425 }
426 })
427 }
428}
429
430func TestR2BindingRefusalKeepsPromotedFix(t *testing.T) {
431 if !plan14RootFixture(t) {
432 return
433 }
434 for _, during := range []bool{false, true} {
435 t.Run(map[bool]string{false: "before restart", true: "after restart"}[during], func(t *testing.T) {
436 f := r2Fixture(t, "list")
437 if during {
438 f.u.afterRestart = func() { f.source.namedPath = filepath.Join(t.TempDir(), "missing") }
439 } else {
440 f.source.unitDirs = []string{"/wrong"}
441 }
442 if err := f.pass(t); err != nil {
443 t.Fatal(err)
444 }
445 f.wantLive(t, "v2")
446 f.wantRunning(t, "v2")
447 })
448 }
449}
450
451func TestR2CrashBeforeRestartCapture(t *testing.T) {
452 if !plan14RootFixture(t) {
453 return
454 }
455 // Merge ruling 6: without a census the rollback target is main's cur
456 // (the interrupted repoint's v2); only a census-backed box retains the
457 // verified running build (v1) as its rollback target.
458 for _, mode := range []string{"no census", "census"} {
459 for _, fail := range []bool{false, true} {
460 t.Run(mode+"/"+map[bool]string{false: "forward", true: "rollback"}[fail], func(t *testing.T) {
461 f := r2Fixture(t, "list")
462 if mode == "census" {
463 f.c.legacy = func() (bool, error) { return false, nil }
464 }
465 // Main's rollback target, and the retained one with a census.
466 target := map[string]string{"no census": "v2", "census": "v1"}[mode]
467 crashed := errors.New("crash after repoint before restart capture")
468 func() {
469 defer func() {
470 if recover() != crashed {
471 t.Fatal("missing crash")
472 }
473 }()
474 _ = release.Pass(t.Context(), f.e.root.reg, f.e.store, f.pub, f.u.opt, func(string) error { panic(crashed) }, release.WithRunningVersion(f.c.runningVersion))
475 }()
476 f.wantLive(t, "v2")
477 f.wantRunning(t, "v1")
478 state, err := f.c.attempts()
479 if err != nil {
480 t.Fatal(err)
481 }
482 if _, err := readRollbackAttempt(state, f.rt.Host); !os.IsNotExist(err) {
483 t.Fatal("attempt already captured", err)
484 }
485 m := release.Manifest{App: f.rt.App, Version: "v3", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}}
486 if err := release.Ship(t.Context(), f.e.store, "bab", m, map[string][]byte{release.AdoptPlatform(): []byte("third build")}); err != nil {
487 t.Fatal(err)
488 }
489 f.promote(t, "v3")
490 // A new resolver and restart callback model the next root process.
491 f.source.index = nil
492 if fail {
493 f.u.afterEffect = func() {
494 if f.u.exe == release.VersionPath(f.u.opt, "", "v3", f.rt.App) {
495 f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = "failed", "failed", 0
496 }
497 }
498 }
499 err = f.pass(t)
500 if fail {
501 if err == nil {
502 t.Fatal("failed build accepted")
503 }
504 f.wantLive(t, target)
505 f.wantRunning(t, target)
506 } else {
507 if err != nil {
508 t.Fatal(err)
509 }
510 f.wantLive(t, "v3")
511 f.wantRunning(t, "v3")
512 prior, err := os.Readlink(filepath.Join(f.u.opt, "previous", f.rt.Host))
513 if err != nil || prior != release.VersionPath(f.u.opt, "", target, f.rt.App) {
514 t.Fatal("previous pointer", prior, err)
515 }
516 pins, err := release.LivePins(f.u.opt)
517 if err != nil || !pins[target] {
518 t.Fatal("rollback build not pinned", pins, err)
519 }
520 }
521 })
522 }
523 }
524}
525
526func TestR2RetainedCensusKeepsStrictRestart(t *testing.T) {
527 if !plan14RootFixture(t) {
528 return
529 }
530 for _, body := range []string{"{}", "corrupt"} {
531 t.Run(body, func(t *testing.T) {
532 f := r2Fixture(t, "none")
533 s, err := rootstate.Open(rootparticipation.Namespace, true, false)
534 if err != nil {
535 t.Fatal(err)
536 }
537 if err := s.Write("census.json", []byte(body)); err != nil {
538 t.Fatal(err)
539 }
540 defer func() { s.Remove("census.json"); s.Close() }()
541 if legacy, err := legacyRootRestart(); err != nil || legacy {
542 t.Fatal("retained census selected legacy", legacy, err)
543 }
544 if err := f.restart()(f.rt.Host); err == nil || len(f.u.effects) != 0 {
545 t.Fatal("unbound restart passed retained census", err, f.u.effects)
546 }
547 })
548 }
549}
550
551func TestR2UnguardedCrashRollback(t *testing.T) {
552 if !plan14RootFixture(t) {
553 return
554 }
555 f := r2Fixture(t, "none")
556 f.u.afterEffect = func() {
557 if f.u.exe == release.VersionPath(f.u.opt, "", "v2", f.rt.App) {
558 f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = "failed", "failed", 0
559 }
560 }
561 if err := f.pass(t); err == nil {
562 t.Fatal("crashed build accepted")
563 }
564 f.wantLive(t, "v1")
565 f.wantRunning(t, "v1")
566 if len(f.u.effects) != 3 {
567 t.Fatal("rollback did not reset and restart", f.u.effects)
568 }
569}
570
571func TestR2NamedFallbackMatchesMain(t *testing.T) {
572 if !plan14RootFixture(t) {
573 return
574 }
575 for _, scenario := range []string{"show error", "exec mismatch", "inactive", "failed", "activating", "try-restart error"} {
576 t.Run(scenario, func(t *testing.T) {
577 f := r2Fixture(t, "none")
578 switch scenario {
579 case "show error":
580 f.u.showErr = os.ErrPermission
581 case "exec mismatch":
582 f.u.props.ExecPath = "/other"
583 case "inactive", "failed", "activating":
584 f.u.props.ActiveState = scenario
585 f.u.props.MainPID = 0
586 case "try-restart error":
587 f.u.tryErr = os.ErrPermission
588 }
589 err := f.restart()(f.rt.Host)
590 if (scenario == "try-restart error") != errors.Is(err, os.ErrPermission) {
591 t.Fatal("fallback changed try-restart result", err)
592 }
593 if len(f.u.tries) != 1 || f.u.tries[0] != f.rt.Unit {
594 t.Fatal(f.u.tries)
595 }
596 if (scenario == "inactive" || scenario == "failed") && len(f.u.effects) != 0 {
597 t.Fatal("try-restart woke stopped unit")
598 }
599 if scenario == "try-restart error" {
600 starts := 0
601 f.c.legacyStart = func(unit string) error {
602 if unit != f.rt.Unit {
603 t.Fatal(unit)
604 }
605 starts++
606 return nil
607 }
608 if err := rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, nil, f.c)(f.rt.Host); err != nil || starts != 1 {
609 t.Fatal("non-NotUp legacy recovery did not use main start", err, starts)
610 }
611 }
612 })
613 }
614}
615