| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os/exec" |
| 7 | "reflect" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/activator" |
| 12 | "github.com/carlosframework/platform/internal/hostbox" |
| 13 | "github.com/carlosframework/platform/internal/hostclaim" |
| 14 | "github.com/carlosframework/platform/internal/registry" |
| 15 | ) |
| 16 | |
| 17 | func TestFilesSeedPreparationOrdersAndSettles(t *testing.T) { |
| 18 | for _, failure := range []string{"none", "prerequisites", "admit", "prepare", "restore", "reserve", "materialize", "settle"} { |
| 19 | t.Run(failure, func(t *testing.T) { |
| 20 | var events []string |
| 21 | mark := func(s string) error { |
| 22 | events = append(events, s) |
| 23 | if failure == s { |
| 24 | return hostbox.ErrChanged |
| 25 | } |
| 26 | return nil |
| 27 | } |
| 28 | obs := hostclaim.Observed{Host: recoveryHost, Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{Incarnation: "inc"}}} |
| 29 | svc := &activator.Service{} |
| 30 | token, err := svc.AcquireSeedExclusion(t.Context(), obs.Host, nil) |
| 31 | if err != nil { |
| 32 | t.Fatal(err) |
| 33 | } |
| 34 | r := &activator.ClaimRuntime{ |
| 35 | Read: func(context.Context, string) (hostclaim.Observed, error) { return obs, nil }, |
| 36 | Prerequisites: func(context.Context, registry.Route, hostclaim.Observed, activator.StartPolicy) error { |
| 37 | return mark("prerequisites") |
| 38 | }, |
| 39 | Admit: func(context.Context, registry.Route, hostclaim.Observed, activator.StartPolicy) (hostclaim.Observed, error) { |
| 40 | return obs, mark("admit") |
| 41 | }, |
| 42 | FailedAdmission: func(ctx context.Context, _ hostclaim.Observed, _ error) error { |
| 43 | if ctx.Err() != nil { |
| 44 | t.Fatal("canceled cleanup") |
| 45 | } |
| 46 | return mark("cleanup") |
| 47 | }, |
| 48 | Prepare: func(context.Context, registry.Route, hostclaim.Observed, activator.StartPolicy) (activator.ClaimStart, error) { |
| 49 | if err := mark("prepare"); err != nil { |
| 50 | return activator.ClaimStart{}, err |
| 51 | } |
| 52 | return activator.ClaimStart{Restore: func(context.Context) error { return mark("restore") }, Reserve: func(_ context.Context, n int) error { |
| 53 | if n != 0 { |
| 54 | t.Fatal("additional attempt") |
| 55 | } |
| 56 | return mark("reserve") |
| 57 | }, Materialize: func(context.Context) error { return mark("materialize") }, Failed: func(context.Context, int) error { return mark("settle") }, Build: func() *exec.Cmd { t.Fatal("seed launched tenant"); return nil }, Healthy: func(context.Context, int, *exec.Cmd) error { t.Fatal("seed became healthy"); return nil }}, nil |
| 58 | }, |
| 59 | } |
| 60 | _, settled, err := prepareFilesSeed(t.Context(), registry.Route{Host: obs.Host}, r, obs, token) |
| 61 | if failure == "none" { |
| 62 | if err != nil || !settled { |
| 63 | t.Fatal(err, settled) |
| 64 | } |
| 65 | want := []string{"prerequisites", "admit", "prepare", "restore", "reserve", "materialize", "settle", "cleanup"} |
| 66 | if !reflect.DeepEqual(events, want) { |
| 67 | t.Fatal(events) |
| 68 | } |
| 69 | } else if err == nil || settled == (failure == "settle") { |
| 70 | t.Fatal("wrong cleanup outcome", events, err, settled) |
| 71 | } |
| 72 | if err := token.Check(obs.Host); err != nil { |
| 73 | t.Fatal("driver released caller's fence", err) |
| 74 | } |
| 75 | }) |
| 76 | } |
| 77 | } |
| 78 | func TestFilesSeedCommandRefusesBeforeConstruction(t *testing.T) { |
| 79 | for _, args := range [][]string{{}, {"repair"}, {"seed", "--preflight"}, {"seed", "--host", "../bad"}, {"seed", "extra"}} { |
| 80 | if _, err := parseFilesSeed(args); err == nil { |
| 81 | t.Fatal(args) |
| 82 | } |
| 83 | } |
| 84 | host, err := parseFilesSeed([]string{"seed", "--host", recoveryHost}) |
| 85 | if err != nil || host != recoveryHost { |
| 86 | t.Fatal(host, err) |
| 87 | } |
| 88 | t.Setenv("CARLOS_HOST_CLAIM", "") |
| 89 | t.Setenv("CARLOS_HOST_CLAIM_ORIGINS", "") |
| 90 | if err := runSystemFiles(t.Context(), []string{"seed"}, nil); err == nil { |
| 91 | t.Fatal("flags off accepted") |
| 92 | } |
| 93 | t.Setenv("CARLOS_HOST_CLAIM", "1") |
| 94 | t.Setenv("CARLOS_HOST_CLAIM_ORIGINS", "1") |
| 95 | if err := runSystemFiles(t.Context(), []string{"seed"}, nil); !errors.Is(err, errFilesSeedDelegation) { |
| 96 | t.Fatal("shell reached construction", err) |
| 97 | } |
| 98 | } |
| 99 | |
| 100 | func TestFilesSeedRootCapabilityRequiresDelegationValidator(t *testing.T) { |
| 101 | f := newLaunchFixture(t, hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box-1"}) |
| 102 | q := hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchCapability, Kind: "files-seed"} |
| 103 | if _, err := f.root.Serve(t.Context(), f.peer, q); err == nil { |
| 104 | t.Fatal("unbound validator accepted") |
| 105 | } |
| 106 | called := 0 |
| 107 | f.root.SeedController = func(_ context.Context, p StopPeer) error { |
| 108 | called++ |
| 109 | if p != f.peer { |
| 110 | t.Fatal("caller invented peer") |
| 111 | } |
| 112 | return hostbox.ErrChanged |
| 113 | } |
| 114 | if _, err := f.root.Serve(t.Context(), f.peer, q); !errors.Is(err, hostbox.ErrChanged) || called != 1 { |
| 115 | t.Fatal(err, called) |
| 116 | } |
| 117 | f.root.SeedController = func(context.Context, StopPeer) error { return nil } |
| 118 | if _, err := f.root.Serve(t.Context(), f.peer, q); err != nil { |
| 119 | t.Fatal(err) |
| 120 | } |
| 121 | if len(f.units.starts) != 0 { |
| 122 | t.Fatal("capability launched tenant") |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | func TestFilesSeedAmbiguousAdmissionRetainsExclusion(t *testing.T) { |
| 127 | before := hostclaim.Observed{Host: recoveryHost, Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{Incarnation: "inc"}}} |
| 128 | after := hostclaim.Observed{Host: recoveryHost, Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{Incarnation: "inc", WakeRetry: &hostclaim.WakeRetry{State: "attempting"}}}} |
| 129 | service := &activator.Service{} |
| 130 | token, err := service.AcquireSeedExclusion(t.Context(), recoveryHost, nil) |
| 131 | if err != nil { |
| 132 | t.Fatal(err) |
| 133 | } |
| 134 | runtime := &activator.ClaimRuntime{ |
| 135 | Read: func(context.Context, string) (hostclaim.Observed, error) { return after, nil }, |
| 136 | Prerequisites: func(context.Context, registry.Route, hostclaim.Observed, activator.StartPolicy) error { return nil }, |
| 137 | Admit: func(context.Context, registry.Route, hostclaim.Observed, activator.StartPolicy) (hostclaim.Observed, error) { |
| 138 | return hostclaim.Observed{}, hostbox.ErrChanged |
| 139 | }, |
| 140 | Prepare: func(context.Context, registry.Route, hostclaim.Observed, activator.StartPolicy) (activator.ClaimStart, error) { |
| 141 | t.Fatal("ambiguous admission prepared") |
| 142 | return activator.ClaimStart{}, nil |
| 143 | }, |
| 144 | FailedAdmission: func(context.Context, hostclaim.Observed, error) error { return nil }, |
| 145 | } |
| 146 | _, settled, err := prepareFilesSeed(t.Context(), registry.Route{Host: recoveryHost}, runtime, before, token) |
| 147 | if err == nil || settled || token.Check(recoveryHost) != nil { |
| 148 | t.Fatal("ambiguous debt reported settled", err, settled) |
| 149 | } |
| 150 | } |
| 151 | |
| 152 | func TestFilesSeedExitCodes(t *testing.T) { |
| 153 | if filesSeedExitCode(nil) != 0 || filesSeedExitCode(hostbox.ErrChanged) != 1 || filesSeedExitCode(filesSeedIncompleteError{hostbox.ErrChanged}) != 2 { |
| 154 | t.Fatal("incorrect status") |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | func TestFilesSeedPreparationCleanupSharesCanceledBudget(t *testing.T) { |
| 159 | ctx, cancel := context.WithCancel(t.Context()) |
| 160 | ctx, finish := activator.WithCleanupBudget(ctx, 50*time.Millisecond) |
| 161 | defer finish() |
| 162 | obs := hostclaim.Observed{Host: recoveryHost, Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{Incarnation: "inc"}}} |
| 163 | token, err := (&activator.Service{}).AcquireSeedExclusion(ctx, recoveryHost, nil) |
| 164 | if err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | var deadline time.Time |
| 168 | runtime := &activator.ClaimRuntime{ |
| 169 | Read: func(ctx context.Context, _ string) (hostclaim.Observed, error) { return obs, ctx.Err() }, |
| 170 | Prerequisites: func(context.Context, registry.Route, hostclaim.Observed, activator.StartPolicy) error { return nil }, |
| 171 | Admit: func(context.Context, registry.Route, hostclaim.Observed, activator.StartPolicy) (hostclaim.Observed, error) { |
| 172 | return obs, nil |
| 173 | }, |
| 174 | Prepare: func(context.Context, registry.Route, hostclaim.Observed, activator.StartPolicy) (activator.ClaimStart, error) { |
| 175 | return activator.ClaimStart{Restore: func(context.Context) error { return nil }, Reserve: func(context.Context, int) error { return nil }, Materialize: func(context.Context) error { cancel(); return context.Canceled }, Failed: func(ctx context.Context, _ int) error { deadline, _ = ctx.Deadline(); <-ctx.Done(); return ctx.Err() }}, nil |
| 176 | }, |
| 177 | FailedAdmission: func(ctx context.Context, _ hostclaim.Observed, _ error) error { |
| 178 | d, _ := ctx.Deadline() |
| 179 | if !d.Equal(deadline) || ctx.Err() == nil { |
| 180 | t.Error("cleanup received fresh budget") |
| 181 | } |
| 182 | return ctx.Err() |
| 183 | }, |
| 184 | } |
| 185 | started := time.Now() |
| 186 | _, settled, err := prepareFilesSeed(ctx, registry.Route{Host: recoveryHost}, runtime, obs, token) |
| 187 | if err == nil || settled || time.Since(started) > time.Second || token.Check(recoveryHost) != nil { |
| 188 | t.Fatal("unbounded or falsely settled cleanup", err, settled) |
| 189 | } |
| 190 | } |
| 191 | |