| 1 | //go:build linux && rootrunner |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "github.com/carlosframework/platform/internal/instancectl" |
| 10 | "github.com/carlosframework/platform/internal/rootstate" |
| 11 | "os" |
| 12 | "path/filepath" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/hostbox" |
| 17 | "github.com/carlosframework/platform/internal/hostguard" |
| 18 | "github.com/carlosframework/platform/internal/provision" |
| 19 | "github.com/carlosframework/platform/internal/registry" |
| 20 | ) |
| 21 | |
| 22 | func TestPlan14CombinedBoundRetirementRootRunner(t *testing.T) { |
| 23 | for _, unit := range []string{"tcp", "socket-program", "backing-program"} { |
| 24 | t.Run(unit, func(t *testing.T) { plan14InstalledClaimLaunch(t, "combined:materialized:"+unit) }) |
| 25 | } |
| 26 | } |
| 27 | |
| 28 | // Installation, admission and the completed receipt are fixture authority. |
| 29 | // The caller has already joined the real controller and stopped both real unit |
| 30 | // invocations. This adds the production retirement composition and registry. |
| 31 | func plan14CombinedBoundRetirement(t *testing.T, dir string, f *launchFixture, m *claimUnitMaterializer, startup bool) (func() error, func()) { |
| 32 | t.Helper() |
| 33 | t.Setenv("CARLOS_HOST_CLAIM", "true") |
| 34 | target := f.target |
| 35 | target.FenceID = "combined-retirement" |
| 36 | observed, err := f.claims.Read(t.Context(), target.Host) |
| 37 | if err != nil { |
| 38 | t.Fatal(err) |
| 39 | } |
| 40 | target.ClaimETag, target.ClaimGen = observed.ETag, observed.Claim.Gen |
| 41 | if target, err = f.root.Guard.BeginFence(t.Context(), target); err != nil { |
| 42 | t.Fatal("combined fence", err) |
| 43 | } |
| 44 | at := time.Now().UTC() |
| 45 | receipt := StopReceipt{V: 1, Kind: "stopped", ID: "combined-receipt", FenceID: target.FenceID, Host: target.Host, Incarnation: target.Incarnation, InstallationStamp: target.Stamp, WriterSession: target.Session, Region: target.Region, ClaimETag: target.ClaimETag, ClaimGen: target.ClaimGen, OperationID: target.Op, Phase: target.Phase, ObligationID: target.ObligationID, Box: target.Box, Enrollment: target.Enrollment, BootID: target.BootID, ControllerID: target.ControllerID, DirectoryRevision: testReceiptNumeral(target), Binding: target.DirectoryRevision, EvidenceAt: at} |
| 46 | raw, _ := json.Marshal(receipt) |
| 47 | if err = f.root.Guard.CompleteFence(t.Context(), target, raw, at, func(ctx context.Context, t hostguard.FenceTarget, body []byte, when time.Time) error { |
| 48 | return verifyRetirementReceipt(ctx, hostguard.FenceReceipt{Target: t, Bytes: body, EvidenceAt: when}) |
| 49 | }); err != nil { |
| 50 | t.Fatal(err) |
| 51 | } |
| 52 | reg, err := registry.Open(filepath.Join(dir, "combined-registry.db")) |
| 53 | if err != nil { |
| 54 | t.Fatal(err) |
| 55 | } |
| 56 | t.Cleanup(func() { reg.Close() }) |
| 57 | inventory, err := openRebootInventory(t, filepath.Join(dir, "combined-inventory.db")) |
| 58 | if err != nil { |
| 59 | t.Fatal(err) |
| 60 | } |
| 61 | t.Cleanup(func() { inventory.Close() }) |
| 62 | b, err := f.root.unitBindings.read(target.Host) |
| 63 | if err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | row := b.Shape.route() |
| 67 | row.ClaimOwnership = registry.ClaimOwnership{ClaimIncarnation: target.Incarnation, ClaimStamp: target.Stamp, ClaimOwner: "combined-owner", ClaimRowID: "combined-route"} |
| 68 | if err = reg.Upsert(row); err != nil { |
| 69 | t.Fatal(err) |
| 70 | } |
| 71 | timer := registry.Timer{Host: target.Host, Name: "backup", Kind: "cron", Registration: "timer-1", ClaimOwnership: row.ClaimOwnership} |
| 72 | timer.ClaimRowID = "timer-1" |
| 73 | if err = reg.ReplaceTimers(target.Host, []registry.Timer{timer}); err != nil { |
| 74 | t.Fatal(err) |
| 75 | } |
| 76 | in := hostbox.StopInventory{Complete: true, Host: target.Host, Incarnation: target.Incarnation, InstallationStamp: target.Stamp, Box: target.Box, Enrollment: target.Enrollment, DirectoryRevision: target.DirectoryRevision, Entries: []hostbox.StopInventoryEntry{ |
| 77 | {Kind: "route", ArtifactID: "route", ManagedRoot: "registry", RelativePath: target.Host, RowIdentity: row.ClaimRowID, Ownership: row.ClaimOwner, Materialization: "materialized"}, |
| 78 | {Kind: "timer", ArtifactID: "timer", ManagedRoot: "registry", RelativePath: target.Host + "/backup", RowIdentity: "timer-1", Ownership: row.ClaimOwner, Materialization: "materialized"}, |
| 79 | }} |
| 80 | if err = inventory.WriteStopInventory(t.Context(), in); err != nil { |
| 81 | t.Fatal(err) |
| 82 | } |
| 83 | journal := filepath.Join(dir, "combined-retirement") |
| 84 | root := ClaimRootHTTP{Stop: ClaimStopHTTP{Coordinator: ClaimStopCoordinator{Guard: f.root.Guard}}, Roots: map[string]string{"data": "/data"}, Reconcile: hostbox.ReconcileAdapter{Inventory: inventory, Registry: reg, Retirement: hostbox.TypedRetirement{JournalDir: journal}}, retireBoundUnit: func(ctx context.Context, t hostguard.FenceTarget) error { |
| 85 | return retireClaimBoundUnit(ctx, t, f.root.unitBindings, *m, f.root.Units.Gone) |
| 86 | }} |
| 87 | crashed := false |
| 88 | replace := m.replace |
| 89 | if startup { |
| 90 | m.replace = func(unit string, id rootstate.FileIdentity, raw []byte) error { |
| 91 | if !crashed { |
| 92 | crashed = true |
| 93 | return os.ErrClosed |
| 94 | } |
| 95 | return replace(unit, id, raw) |
| 96 | } |
| 97 | } |
| 98 | restarted := false |
| 99 | retire := func() error { |
| 100 | err := root.retire(t.Context(), target) |
| 101 | if startup && crashed && !restarted { |
| 102 | if !errors.Is(err, os.ErrClosed) { |
| 103 | t.Fatal("crash boundary", err) |
| 104 | } |
| 105 | partial, e := m.load(target.Host) |
| 106 | if e != nil || !partial.Retiring || partial.Retired { |
| 107 | t.Fatal("missing retirement intention", e) |
| 108 | } |
| 109 | oldOpen, oldMaterials, oldShapes := openClaimUnitBindings, openClaimUnitMaterializations, productionClaimUnitShapes |
| 110 | defer func() { |
| 111 | openClaimUnitBindings, openClaimUnitMaterializations, productionClaimUnitShapes = oldOpen, oldMaterials, oldShapes |
| 112 | }() |
| 113 | openClaimUnitBindings = func() (rootstate.Store, error) { |
| 114 | return rootstate.Open(filepath.Join(dir, "unit-bindings"), false, false) |
| 115 | } |
| 116 | openClaimUnitMaterializations = func() (rootstate.Store, error) { |
| 117 | return rootstate.Open(filepath.Join(dir, "unit-materializations"), false, false) |
| 118 | } |
| 119 | productionClaimUnitShapes = func() (map[string]claimUnitShape, error) { return nil, os.ErrPermission } |
| 120 | copyLaunch := *f.root |
| 121 | copyLaunch.Units = f.root.Units.(claimUnitDispatch).base |
| 122 | copyRoot := root |
| 123 | copyRoot.Stop.Coordinator.Kernel = claimUnitStopKernel{Units: copyLaunch.Units.(instancectl.ClaimUnits)} |
| 124 | wrapped := ClaimRootWithLaunch{Root: copyRoot, Launch: ©Launch} |
| 125 | assembly := &ClaimAssembly{} |
| 126 | if e = bindProductionClaimUnits(t.Context(), &wrapped, assembly); e != nil { |
| 127 | t.Fatal("startup crash replay", e) |
| 128 | } |
| 129 | defer assembly.Close(t.Context()) |
| 130 | saved, e := wrapped.Launch.unitBindings.read(target.Host) |
| 131 | if e != nil || !saved.Retired { |
| 132 | t.Fatal("startup did not finish bound retirement", saved, e) |
| 133 | } |
| 134 | if _, _, e = claimLaunchPlacement(wrapped.Launch.Units, "ordinary.example"); e != nil { |
| 135 | t.Fatal("ordinary host blocked", e) |
| 136 | } |
| 137 | restarted = true |
| 138 | err = nil |
| 139 | } |
| 140 | records, e := generationRetirementRecords(t.Context(), &rollbackDisk{dir: journal}) |
| 141 | if e == nil && len(records) == 1 && records[0].Complete { |
| 142 | return nil |
| 143 | } |
| 144 | return err |
| 145 | } |
| 146 | verify := func() { |
| 147 | got, e := reg.Get(target.Host) |
| 148 | if e != nil || got != nil { |
| 149 | t.Fatal("route not retired", got, e) |
| 150 | } |
| 151 | timers, e := reg.TimersFor(target.Host) |
| 152 | if e != nil || len(timers) != 0 { |
| 153 | t.Fatal("timer not retired", timers, e) |
| 154 | } |
| 155 | for _, leaf := range []string{"launch.json", "permit.json", "exec.sock"} { |
| 156 | if _, e := os.Lstat(filepath.Join("/run/carlos-control/exe/.claim", provision.InstanceID(target.Host), leaf)); !os.IsNotExist(e) { |
| 157 | t.Fatal("runtime environment retained", leaf, e) |
| 158 | } |
| 159 | } |
| 160 | before, e := os.ReadFile(filepath.Join(journal, target.FenceID+".json")) |
| 161 | if e != nil { |
| 162 | t.Fatal(e) |
| 163 | } |
| 164 | row.ClaimStamp++ |
| 165 | row.ClaimRowID = "successor-route" |
| 166 | row.ClaimOwner = "successor-owner" |
| 167 | timer.ClaimOwnership = row.ClaimOwnership |
| 168 | timer.ClaimRowID = "successor-timer" |
| 169 | timer.Registration = "successor-timer" |
| 170 | if e = reg.Upsert(row); e != nil { |
| 171 | t.Fatal(e) |
| 172 | } |
| 173 | if e = reg.ReplaceTimers(target.Host, []registry.Timer{timer}); e != nil { |
| 174 | t.Fatal(e) |
| 175 | } |
| 176 | if e = retire(); e != nil { |
| 177 | t.Fatal("local completed replay", e) |
| 178 | } |
| 179 | got, e = reg.Get(target.Host) |
| 180 | if e != nil || got == nil || got.ClaimRowID != row.ClaimRowID { |
| 181 | t.Fatal("successor route changed", e) |
| 182 | } |
| 183 | timers, e = reg.TimersFor(target.Host) |
| 184 | if e != nil || len(timers) != 1 || timers[0].Registration != timer.Registration { |
| 185 | t.Fatal("successor timer changed", e) |
| 186 | } |
| 187 | after, e := os.ReadFile(filepath.Join(journal, target.FenceID+".json")) |
| 188 | if e != nil || string(before) != string(after) { |
| 189 | t.Fatal("completed retirement changed", e) |
| 190 | } |
| 191 | } |
| 192 | return retire, verify |
| 193 | } |
| 194 | |
| 195 | func TestPlan14BoundStartupRetirementRootRunner(t *testing.T) { |
| 196 | plan14InstalledClaimLaunch(t, "startup:combined:materialized:tcp") |
| 197 | } |
| 198 | |
amadan