| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os/exec" |
| 7 | "reflect" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/hostbox" |
| 13 | "github.com/carlosframework/platform/internal/hostclaim" |
| 14 | "github.com/carlosframework/platform/internal/hostwrite" |
| 15 | "github.com/carlosframework/platform/internal/lease" |
| 16 | "github.com/carlosframework/platform/internal/registry" |
| 17 | "github.com/carlosframework/platform/internal/release" |
| 18 | ) |
| 19 | |
| 20 | func plan11Relocating(phase string) hostclaim.Observed { |
| 21 | exec := "box-2" |
| 22 | switch phase { |
| 23 | case "starting", "releasing": |
| 24 | exec = "box-1" |
| 25 | case "pinning": |
| 26 | exec = "none" |
| 27 | } |
| 28 | return hostclaim.Observed{Host: recoveryHost, Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{Session: 9, WriterBox: "box-2"}, |
| 29 | Op: hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "r1", Phase: phase, Exec: exec, Participants: []string{"box-1", "box-2"}, |
| 30 | Intent: map[string]string{"region": "ie", "source_box": "box-1", "target_box": "box-2"}}}} |
| 31 | } |
| 32 | |
| 33 | func plan11Env(moves, ha string) func(string) string { |
| 34 | return func(k string) string { |
| 35 | switch k { |
| 36 | case "CARLOS_HOST_CLAIM": |
| 37 | return "1" |
| 38 | case "CARLOS_MOVES_PHASE1": |
| 39 | return moves |
| 40 | case "CARLOS_REGION_HA": |
| 41 | return ha |
| 42 | } |
| 43 | return "" |
| 44 | } |
| 45 | } |
| 46 | |
| 47 | func plan11RelocationOwners(calls *[]string) ClaimRelocationOwners { |
| 48 | return ClaimRelocationOwners{BeforeAdmission: func(context.Context, hostclaim.Observed) error { return nil }, AfterAdmission: func(context.Context, hostclaim.Observed, hostclaim.Observed) error { return nil }, ClaimKindOwners: plan9D5Owner{"relocation", calls}.owners(), Operations: &plan9D2Attempts{}, |
| 49 | Policy: func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }} |
| 50 | } |
| 51 | |
| 52 | // R1 as accepted: ClaimOwnerDispatch's typed Relocation entry receives |
| 53 | // relocating observations only at a phase of the relocating session row |
| 54 | // (probing, and the probe's session carried into serving). Unbound, the |
| 55 | // kind keeps main's refusal; an incomplete entry is a typed refusal naming |
| 56 | // region HA; any other relocating phase reaches no owner. |
| 57 | func TestPlan11OwnerDispatchRoutesRelocatingToItsTypedEntry(t *testing.T) { |
| 58 | var calls []string |
| 59 | moves := plan9D5Owner{"moves", &calls}.owners() |
| 60 | relocation := plan9D5Owner{"relocation", &calls}.owners() |
| 61 | d := ClaimOwnerDispatch{Serving: plan9D5Owner{"serving", &calls}.owners(), Moves: &moves, Relocation: &relocation} |
| 62 | all := "select,materialize,empty,publish,admission,publication" |
| 63 | want := "relocation." + strings.ReplaceAll(all, ",", ",relocation.") |
| 64 | for _, phase := range []string{"probing", "serving"} { |
| 65 | calls = nil |
| 66 | if err := plan9D5Call(d, plan11Relocating(phase)); err != nil { |
| 67 | t.Fatalf("%s: %v", phase, err) |
| 68 | } |
| 69 | if got := strings.Join(calls, ","); got != want { |
| 70 | t.Fatalf("%s routed %s", phase, got) |
| 71 | } |
| 72 | } |
| 73 | for _, phase := range []string{"starting", "releasing", "pinning", "installing", "installed", "probe-failed"} { |
| 74 | calls = nil |
| 75 | if err := plan9D5Call(d, plan11Relocating(phase)); !errors.Is(err, hostbox.ErrChanged) || len(calls) != 0 { |
| 76 | t.Fatalf("%s: %v %v", phase, err, calls) |
| 77 | } |
| 78 | } |
| 79 | // Every move kind still goes to moves, serving to serving. |
| 80 | for kind, owner := range map[string]string{hostclaim.KindServing: "serving", hostclaim.KindLanding: "moves", hostclaim.KindRestoring: "moves"} { |
| 81 | calls = nil |
| 82 | if err := plan9D5Call(d, plan9D5Kind(kind)); err != nil || !strings.HasPrefix(strings.Join(calls, ","), owner+".select") { |
| 83 | t.Fatalf("%s: %v %v", kind, err, calls) |
| 84 | } |
| 85 | } |
| 86 | // Unbound: main's ErrUnsupported, exactly as before Plan 11. |
| 87 | d.Relocation = nil |
| 88 | calls = nil |
| 89 | if err := plan9D5Call(d, plan11Relocating("probing")); !errors.Is(err, hostbox.ErrUnsupported) || len(calls) != 0 { |
| 90 | t.Fatalf("unbound: %v %v", err, calls) |
| 91 | } |
| 92 | // Incomplete: a typed refusal naming region HA's provider. |
| 93 | d.Relocation = &ClaimKindOwners{Inputs: relocation.Inputs} |
| 94 | var refusal *ClaimDependencyError |
| 95 | if err := plan9D5Call(d, plan11Relocating("probing")); !errors.As(err, &refusal) || !strings.Contains(refusal.Error(), "region HA") || len(calls) != 0 { |
| 96 | t.Fatalf("incomplete: %v %v", err, calls) |
| 97 | } |
| 98 | // The fenced publisher routes to the relocation inputs' own, never to |
| 99 | // the unfenced Publish. |
| 100 | d.Relocation = &relocation |
| 101 | if _, err := d.PublishFenced(context.Background(), plan11Relocating("probing"), ClaimLaunchInputs{}, ClaimPublication{}); !errors.As(err, &refusal) || len(calls) != 0 { |
| 102 | t.Fatalf("fenced publication without a fenced publisher: %v %v", err, calls) |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | // R1/R8: the relocation owner set binds only with CARLOS_HOST_CLAIM, |
| 107 | // CARLOS_MOVES_PHASE1 and CARLOS_REGION_HA all on; a partial set refuses the |
| 108 | // assembly rather than half-wiring it. |
| 109 | func TestPlan11RelocationBindsOnlyWithAllFlagsAndACompleteOwnerSet(t *testing.T) { |
| 110 | var calls []string |
| 111 | complete := plan11RelocationOwners(&calls) |
| 112 | for name, tc := range map[string]struct { |
| 113 | getenv func(string) string |
| 114 | movesC1 bool |
| 115 | owners ClaimRelocationOwners |
| 116 | want bool |
| 117 | err error |
| 118 | }{ |
| 119 | "ha-off": {plan11Env("1", ""), true, complete, false, nil}, |
| 120 | "ha-garbage": {plan11Env("1", "maybe"), true, complete, false, nil}, |
| 121 | "moves-c1-off": {plan11Env("", "1"), false, complete, false, nil}, |
| 122 | "no-owners": {plan11Env("1", "1"), true, ClaimRelocationOwners{}, false, nil}, |
| 123 | "complete": {plan11Env("1", "true"), true, complete, true, nil}, |
| 124 | "no-attempts": {plan11Env("1", "1"), true, ClaimRelocationOwners{ClaimKindOwners: complete.ClaimKindOwners, Policy: complete.Policy}, false, hostwrite.ErrUnavailable}, |
| 125 | "no-policy": {plan11Env("1", "1"), true, ClaimRelocationOwners{ClaimKindOwners: complete.ClaimKindOwners, Operations: complete.Operations}, false, hostwrite.ErrUnavailable}, |
| 126 | "no-publication": {plan11Env("1", "1"), true, ClaimRelocationOwners{ClaimKindOwners: ClaimKindOwners{Inputs: complete.Inputs, AdmissionEvidence: complete.AdmissionEvidence}, Operations: complete.Operations, Policy: complete.Policy}, false, hostwrite.ErrUnavailable}, |
| 127 | "probe-only": {plan11Env("1", "1"), true, ClaimRelocationOwners{ProbeReady: func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil }}, false, hostwrite.ErrUnavailable}, |
| 128 | } { |
| 129 | got, err := claimRelocationC1(tc.getenv, tc.movesC1, tc.owners) |
| 130 | if got != tc.want || !errors.Is(err, tc.err) { |
| 131 | t.Errorf("%s: %v %v", name, got, err) |
| 132 | } |
| 133 | } |
| 134 | } |
| 135 | |
| 136 | // The constructor's Group D wiring: with relocation unbound the moves |
| 137 | // attempts, probe check and policy are exactly Plan 9's; bound, a router |
| 138 | // sends landing to moves and relocating to region HA. |
| 139 | func TestPlan11GroupDWiringIsPlan9sWithRelocationUnbound(t *testing.T) { |
| 140 | var calls []string |
| 141 | serving := plan9D5Owner{"serving", &calls}.owners() |
| 142 | movesAttempts := &plan9D2Attempts{} |
| 143 | moves := ClaimMovesOwners{ClaimKindOwners: plan9D5Owner{"moves", &calls}.owners(), Operations: movesAttempts} |
| 144 | off, err := claimGroupDWiring(plan11Env("1", ""), serving, moves, plan11RelocationOwners(&calls)) |
| 145 | if err != nil { |
| 146 | t.Fatal(err) |
| 147 | } |
| 148 | if off.Dispatch.Relocation != nil || off.Relocation != nil || off.Operations != OperationAttempts(movesAttempts) || off.ProbeReady == nil { |
| 149 | t.Fatalf("relocation leaked with region HA off: %+v", off) |
| 150 | } |
| 151 | if _, routed := off.Operations.(claimOperationRouter); routed { |
| 152 | t.Fatal("router bound with region HA off") |
| 153 | } |
| 154 | reloc := plan11RelocationOwners(&calls) |
| 155 | on, err := claimGroupDWiring(plan11Env("1", "1"), serving, moves, reloc) |
| 156 | if err != nil { |
| 157 | t.Fatal(err) |
| 158 | } |
| 159 | if on.Dispatch.Relocation == nil || on.Relocation == nil || on.Dispatch.Moves == nil { |
| 160 | t.Fatalf("relocation not bound: %+v", on) |
| 161 | } |
| 162 | router, ok := on.Operations.(claimOperationRouter) |
| 163 | if !ok || router[hostclaim.KindLanding].Attempts != OperationAttempts(movesAttempts) || router[hostclaim.KindRelocating].Attempts != reloc.Operations { |
| 164 | t.Fatalf("router: %+v", on.Operations) |
| 165 | } |
| 166 | if router[hostclaim.KindRelocating].ProbeReady == nil || router[hostclaim.KindLanding].ProbeReady == nil { |
| 167 | t.Fatal("probe readiness defaults to the socket check") |
| 168 | } |
| 169 | for _, kind := range []string{hostclaim.KindReverting, hostclaim.KindRestoring} { |
| 170 | if router[kind].Attempts != OperationAttempts(movesAttempts) || router[kind].ProbeReady == nil { |
| 171 | t.Fatalf("recovery owner missing with relocation enabled: %s", kind) |
| 172 | } |
| 173 | } |
| 174 | if _, err := claimGroupDWiring(plan11Env("1", "1"), serving, moves, ClaimRelocationOwners{Operations: reloc.Operations}); !errors.Is(err, hostwrite.ErrUnavailable) { |
| 175 | t.Fatalf("partial relocation set: %v", err) |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | // D2 for relocating: the router serves relocating/probing attempts from |
| 180 | // region HA's journal and landing attempts from moves', and refuses any |
| 181 | // other kind before either journal is touched. |
| 182 | func TestPlan11OperationAttemptsRouteByOpKind(t *testing.T) { |
| 183 | landing, relocating := &plan9D2Attempts{}, &plan9D2Attempts{} |
| 184 | r := claimOperationRouter{hostclaim.KindLanding: {Attempts: landing}, hostclaim.KindRelocating: {Attempts: relocating}} |
| 185 | ctx := context.Background() |
| 186 | start := func(o hostclaim.Observed) OperationStart { return OperationStart{Admitted: o} } |
| 187 | reloc := start(plan11Relocating("probing")) |
| 188 | if _, err := r.OpenProbe(ctx, reloc, time.Unix(5, 0)); err != nil { |
| 189 | t.Fatal(err) |
| 190 | } |
| 191 | r.CurrentAttempt(reloc) |
| 192 | if err := errors.Join(r.Reserve(ctx, reloc, 1), r.Failed(ctx, reloc, 1), r.Healthy(ctx, reloc, 2, nil, func(context.Context) error { return nil })); err != nil { |
| 193 | t.Fatal(err) |
| 194 | } |
| 195 | if got := strings.Join(relocating.calls, ","); got != "open,current,reserve-1,failed-1,healthy-2" || len(landing.calls) != 0 { |
| 196 | t.Fatalf("relocating routed %q, landing %v", got, landing.calls) |
| 197 | } |
| 198 | land := start(plan9D5Kind(hostclaim.KindLanding)) |
| 199 | if err := r.Reserve(ctx, land, 0); err != nil || len(landing.calls) != 1 { |
| 200 | t.Fatalf("landing: %v %v", err, landing.calls) |
| 201 | } |
| 202 | var refusal *ClaimDependencyError |
| 203 | for _, o := range []hostclaim.Observed{plan9D5Kind(hostclaim.KindRestoring), plan9D5Kind(hostclaim.KindServing), {Host: recoveryHost}} { |
| 204 | s := start(o) |
| 205 | _, openErr := r.OpenProbe(ctx, s, time.Unix(5, 0)) |
| 206 | err := errors.Join(openErr, r.Reserve(ctx, s, 1), r.Failed(ctx, s, 1), r.Healthy(ctx, s, 1, nil, func(context.Context) error { return nil })) |
| 207 | if !errors.As(err, &refusal) { |
| 208 | t.Fatalf("unrouted kind: %v", err) |
| 209 | } |
| 210 | if got := r.CurrentAttempt(s); !reflect.DeepEqual(got, hostclaim.WakeRetryAttempt{}) { |
| 211 | t.Fatalf("unrouted CurrentAttempt %+v", got) |
| 212 | } |
| 213 | } |
| 214 | if len(landing.calls) != 1 || len(relocating.calls) != 5 { |
| 215 | t.Fatalf("refused kinds reached a journal: %v %v", landing.calls, relocating.calls) |
| 216 | } |
| 217 | } |
| 218 | |
| 219 | // The wake's operation owner and spawn action helpers the assembly uses. |
| 220 | // With Plan 9's single moves journal, only landing has an owner, exactly as |
| 221 | // before; with the router, relocating has region HA's. |
| 222 | func TestPlan11WakeOperationOwnerAndProbeAction(t *testing.T) { |
| 223 | moves := &plan9D2Attempts{} |
| 224 | ready := func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil } |
| 225 | w := &ClaimWakeAssembly{Operations: moves, ProbeReady: ready} |
| 226 | if ops, r := w.operationOwner(plan9D5Kind(hostclaim.KindLanding)); ops != OperationAttempts(moves) || r == nil { |
| 227 | t.Fatal("plan 9 landing owner") |
| 228 | } |
| 229 | for _, o := range []hostclaim.Observed{plan11Relocating("probing"), plan9D5Kind(hostclaim.KindServing), {}} { |
| 230 | if ops, r := w.operationOwner(o); ops != nil || r != nil { |
| 231 | t.Fatalf("%+v has an owner without the router", o.Claim) |
| 232 | } |
| 233 | } |
| 234 | reloc := &plan9D2Attempts{} |
| 235 | w.Operations = claimOperationRouter{hostclaim.KindLanding: {Attempts: moves, ProbeReady: ready}, hostclaim.KindRelocating: {Attempts: reloc, ProbeReady: ready}} |
| 236 | if ops, r := w.operationOwner(plan11Relocating("probing")); ops != OperationAttempts(reloc) || r == nil { |
| 237 | t.Fatal("relocating owner") |
| 238 | } |
| 239 | if ops, _ := w.operationOwner(plan9D5Kind(hostclaim.KindLanding)); ops != OperationAttempts(moves) { |
| 240 | t.Fatal("landing owner through the router") |
| 241 | } |
| 242 | if ops, _ := w.operationOwner(plan9D5Kind(hostclaim.KindReverting)); ops != nil { |
| 243 | t.Fatal("reverting owner") |
| 244 | } |
| 245 | if !w.operationKind(plan11Relocating("probing")) || w.operationKind(plan9D5Kind(hostclaim.KindServing)) { |
| 246 | t.Fatal("operationKind") |
| 247 | } |
| 248 | if got := claimSpawnAction(plan11Relocating("probing").Claim); got != "probe" { |
| 249 | t.Fatalf("relocating spawn action %q", got) |
| 250 | } |
| 251 | for _, kind := range []string{hostclaim.KindServing, hostclaim.KindLanding, hostclaim.KindRestoring} { |
| 252 | if got := claimSpawnAction(plan9D5Kind(kind).Claim); got != "spawn" { |
| 253 | t.Fatalf("%s spawn action %q", kind, got) |
| 254 | } |
| 255 | } |
| 256 | if claimSpawnAction(nil) != "spawn" { |
| 257 | t.Fatal("nil claim") |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | type plan11Source struct { |
| 262 | owners ClaimRelocationOwners |
| 263 | refusals []hostbox.PolicyRefusal |
| 264 | opened int |
| 265 | closed int |
| 266 | err error |
| 267 | } |
| 268 | |
| 269 | func (s *plan11Source) Open(context.Context, release.Store, lease.VersionedStore, hostbox.Actor) (ClaimRelocationOwners, []hostbox.PolicyRefusal, error) { |
| 270 | s.opened++ |
| 271 | return s.owners, s.refusals, s.err |
| 272 | } |
| 273 | func (s *plan11Source) Close(context.Context) error { s.closed++; return nil } |
| 274 | |
| 275 | // Production registration: region HA's source registers once; it is opened |
| 276 | // only with CARLOS_REGION_HA on, and its refusals are appended after any the |
| 277 | // bindings already carry, never replacing them. |
| 278 | func TestPlan11ProductionRelocationSourceRegistersOnceAndOpensOnlyWithRegionHA(t *testing.T) { |
| 279 | claimRelocationSources.Lock() |
| 280 | saved := claimRelocationSources.Source |
| 281 | claimRelocationSources.Source = nil |
| 282 | claimRelocationSources.Unlock() |
| 283 | t.Cleanup(func() { |
| 284 | claimRelocationSources.Lock() |
| 285 | claimRelocationSources.Source = saved |
| 286 | claimRelocationSources.Unlock() |
| 287 | }) |
| 288 | var calls []string |
| 289 | src := &plan11Source{owners: plan11RelocationOwners(&calls)} |
| 290 | if err := RegisterProductionClaimRelocationSource(nil); !errors.Is(err, hostwrite.ErrUnavailable) { |
| 291 | t.Fatalf("nil source: %v", err) |
| 292 | } |
| 293 | if err := RegisterProductionClaimRelocationSource(src); err != nil { |
| 294 | t.Fatal(err) |
| 295 | } |
| 296 | if err := RegisterProductionClaimRelocationSource(&plan11Source{}); !errors.Is(err, hostwrite.ErrUnavailable) || productionClaimRelocationSource() != ClaimProductionRelocationSource(src) { |
| 297 | t.Fatalf("second registration replaced the first: %v", err) |
| 298 | } |
| 299 | ctx := context.Background() |
| 300 | var out ClaimOwnerBindings |
| 301 | closeFn, err := openClaimRelocationOwners(ctx, plan11Env("1", ""), src, nil, nil, hostbox.Actor{}, &out) |
| 302 | if err != nil || closeFn != nil || src.opened != 0 || !out.Relocation.zero() { |
| 303 | t.Fatalf("opened with region HA off: %v %d", err, src.opened) |
| 304 | } |
| 305 | if closeFn, err := openClaimRelocationOwners(ctx, plan11Env("1", "1"), nil, nil, nil, hostbox.Actor{}, &out); err != nil || closeFn != nil { |
| 306 | t.Fatalf("nothing registered: %v", err) |
| 307 | } |
| 308 | first := hostbox.PolicyRefusal(func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }) |
| 309 | added := hostbox.PolicyRefusal(func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }) |
| 310 | out.PolicyRefusals = []hostbox.PolicyRefusal{first} |
| 311 | src.refusals = []hostbox.PolicyRefusal{added} |
| 312 | closeFn, err = openClaimRelocationOwners(ctx, plan11Env("1", "1"), src, nil, nil, hostbox.Actor{}, &out) |
| 313 | if err != nil || closeFn == nil || src.opened != 1 || out.Relocation.Operations == nil || len(out.PolicyRefusals) != 2 { |
| 314 | t.Fatalf("region HA on: %v %d %d", err, src.opened, len(out.PolicyRefusals)) |
| 315 | } |
| 316 | if err := closeFn(ctx); err != nil || src.closed != 1 { |
| 317 | t.Fatal("close is the source's") |
| 318 | } |
| 319 | src.err = errors.New("open failed") |
| 320 | if _, err := openClaimRelocationOwners(ctx, plan11Env("1", "1"), src, nil, nil, hostbox.Actor{}, &ClaimOwnerBindings{}); !errors.Is(err, src.err) { |
| 321 | t.Fatalf("open error: %v", err) |
| 322 | } |
| 323 | } |
| 324 | |
amadan