| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "encoding/json" |
| 8 | "fmt" |
| 9 | "os" |
| 10 | "os/exec" |
| 11 | "path/filepath" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | execlease "github.com/carlosframework/platform/internal/hostexec" |
| 16 | "github.com/carlosframework/platform/internal/provision" |
| 17 | "github.com/carlosframework/platform/internal/release" |
| 18 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 19 | "github.com/carlosframework/platform/internal/rootstate" |
| 20 | ) |
| 21 | |
| 22 | func filesSeedCombinedRecovery(t *testing.T, install func(*servingParkDiskFixture) rootstate.Store) { |
| 23 | t.Helper() |
| 24 | f := newServingParkDiskFixture(t) |
| 25 | state := install(f) |
| 26 | var parks []servingParkRecord |
| 27 | for i := 0; i < 200; i++ { |
| 28 | f.base.Host = fmt.Sprintf("park-%03d.example.com", i) |
| 29 | f.target.Host = f.base.Host |
| 30 | if err := os.MkdirAll(filepath.Dir(provision.NestedDBPath(f.data, f.base.Host)), 0700); err != nil { |
| 31 | t.Fatal(err) |
| 32 | } |
| 33 | f.put("", fmt.Sprintf("retained-%03d", i)) |
| 34 | auth, receipts := f.evidence(10, true, "missing") |
| 35 | r, err := f.park(auth, receipts) |
| 36 | if err != nil { |
| 37 | t.Fatal(i, err) |
| 38 | } |
| 39 | if i != 0 { |
| 40 | f.complete(auth) |
| 41 | } |
| 42 | parks = append(parks, r) |
| 43 | } |
| 44 | ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second) |
| 45 | defer cancel() |
| 46 | maintenance := &servingParkRuntime{authority: &servingParkRoot{ledger: f.ledger}, ctx: ctx, cancel: cancel, idle: make(chan struct{})} |
| 47 | root := ClaimRootHTTP{servingParks: maintenance} |
| 48 | exclusion := &release.ReleaseExclusion{Open: func() (rootstate.Store, error) { return &plan14ReleaseStore{}, nil }} |
| 49 | ledger := rootparticipation.Ledger{State: state, ExcludeLaunches: exclusion.With, VerifyLaunches: func(ctx context.Context, _ uint64) error { return exclusion.Check(ctx) }} |
| 50 | raw, _ := json.Marshal(rootparticipation.Census{Schema: 1, Box: "box", Enrollment: "en", Migration: 1, Revision: 1, LaunchRevision: 1, Complete: true}) |
| 51 | if err := state.Write("census.json", raw); err != nil { |
| 52 | t.Fatal(err) |
| 53 | } |
| 54 | child := exec.Command("sleep", "60") |
| 55 | if err := child.Start(); err != nil { |
| 56 | t.Fatal(err) |
| 57 | } |
| 58 | defer func() { child.Process.Kill(); child.Wait() }() |
| 59 | boot, err := execlease.BootID() |
| 60 | if err != nil { |
| 61 | t.Fatal(err) |
| 62 | } |
| 63 | start, err := claimLaunchProcessStart("/proc", child.Process.Pid) |
| 64 | if err != nil { |
| 65 | t.Fatal(err) |
| 66 | } |
| 67 | peer := execlease.ControllerProcess{KernelBootID: boot, PID: child.Process.Pid, Start: start} |
| 68 | q := execlease.ExecutionRequest{Schema: 1, ID: "begin", Operation: "executionBegin", Host: "crashed.example.com", Incarnation: "inc", Stamp: 1, Session: 1, Op: "move"} |
| 69 | cleaned, settled := 0, 0 |
| 70 | cfg := execlease.RootBrokerConfig{State: state, Remote: release.NewDirStore(t.TempDir()), Clock: execlease.SystemClock{}, BootID: boot, Identity: "old-root", Alive: execlease.ControllerAlive, |
| 71 | Authority: func(context.Context, execlease.ExecutionRequest, execlease.ControllerProcess) (execlease.ExecutionAuthority, error) { |
| 72 | return execlease.ExecutionAuthority{Box: "box", Enrollment: "en", Account: "bab", App: "jam", Installation: "inc/1/box/en", Migration: 1, ClaimRevision: "claim", RecordRevision: "record", DirectoryRevision: "1", Kind: "landing", Phase: "checking"}, nil |
| 73 | }, |
| 74 | Cleanup: func(ctx context.Context, r execlease.ExecutionRecord) error { |
| 75 | if execlease.ControllerAlive(ctx, r.Peer) == nil { |
| 76 | return execlease.ErrState |
| 77 | } |
| 78 | cleaned++ |
| 79 | return nil |
| 80 | }, |
| 81 | Settle: func(context.Context, execlease.ExecutionRecord) error { settled++; return nil }, Exclude: ledger.Acquire, Recovered: ledger.RecoverAcquisition} |
| 82 | broker, err := execlease.NewRootBroker(cfg) |
| 83 | if err != nil { |
| 84 | t.Fatal(err) |
| 85 | } |
| 86 | if _, err := broker.Execute(ctx, peer, q); err != nil { |
| 87 | t.Fatal(err) |
| 88 | } |
| 89 | if err := exclusion.With(ctx, func(ctx context.Context) error { return ledger.BeginUpdate(ctx, "update") }); err == nil { |
| 90 | t.Fatal("live participation allowed update") |
| 91 | } |
| 92 | if err := child.Process.Kill(); err != nil { |
| 93 | t.Fatal(err) |
| 94 | } |
| 95 | child.Wait() |
| 96 | if err := broker.Close(ctx); err != nil { |
| 97 | t.Fatal(err) |
| 98 | } |
| 99 | cleaned = 0 |
| 100 | cfg.Identity = "replacement-root" |
| 101 | broker, err = execlease.NewRootBroker(cfg) |
| 102 | if err != nil { |
| 103 | t.Fatal(err) |
| 104 | } |
| 105 | defer broker.Close(context.Background()) |
| 106 | root.Execution = broker |
| 107 | update, _ := applyHappy(t) |
| 108 | update.a.participation = &ledger |
| 109 | originalRecovery := recoverPendingControllers |
| 110 | t.Cleanup(func() { recoverPendingControllers = originalRecovery }) |
| 111 | recoverPendingControllers = func(call context.Context, _, _ string) error { |
| 112 | return recoverRootExecutionBeforeUpdate(call, broker, root) |
| 113 | } |
| 114 | if err := update.a.controllersRecovered(ctx); err != nil { |
| 115 | t.Fatal(err) |
| 116 | } |
| 117 | if cleaned != 1 || settled != 1 { |
| 118 | t.Fatal("controller debt not settled exactly once", cleaned, settled) |
| 119 | } |
| 120 | if maintenance.recoveryAfter == "" { |
| 121 | t.Fatal("first page lost its cursor") |
| 122 | } |
| 123 | if err := maintenance.recoverPending(ctx); err != nil { |
| 124 | t.Fatal(err) |
| 125 | } |
| 126 | if maintenance.recoveryAfter != "" { |
| 127 | t.Fatal("second page did not complete") |
| 128 | } |
| 129 | if err := exclusion.With(ctx, func(ctx context.Context) error { return ledger.BeginUpdate(ctx, "update") }); err != nil { |
| 130 | t.Fatal("retained parks blocked update", err) |
| 131 | } |
| 132 | if err := exclusion.With(ctx, func(ctx context.Context) error { return ledger.EndUpdateWith(ctx, "update", nil) }); err != nil { |
| 133 | t.Fatal(err) |
| 134 | } |
| 135 | if err := update.a.runWithRelease(ctx, false); err != nil { |
| 136 | t.Fatal(err) |
| 137 | } |
| 138 | if r := update.result(); r == nil || r.Outcome != systemOutcomeApplied { |
| 139 | t.Fatal("update did not apply", r) |
| 140 | } |
| 141 | for i, r := range parks { |
| 142 | f.retained(r.Authorization, fmt.Sprintf("retained-%03d", i), r.ReapAfter) |
| 143 | } |
| 144 | } |
| 145 | |
| 146 | func TestFilesSeedCombinedRecoveryAllowsUpdate(t *testing.T) { |
| 147 | filesSeedCombinedRecovery(t, func(*servingParkDiskFixture) rootstate.Store { |
| 148 | return &servingAuthorityDisk{rollbackDisk: &rollbackDisk{dir: t.TempDir()}} |
| 149 | }) |
| 150 | } |
| 151 | |
amadan