| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "net" |
| 10 | "net/http" |
| 11 | "os" |
| 12 | "os/exec" |
| 13 | "path/filepath" |
| 14 | "sync/atomic" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/hostbox" |
| 19 | "github.com/carlosframework/platform/internal/hostclaim" |
| 20 | "github.com/carlosframework/platform/internal/lease/leasetest" |
| 21 | "github.com/carlosframework/platform/internal/provision" |
| 22 | "github.com/carlosframework/platform/internal/registry" |
| 23 | "github.com/carlosframework/platform/internal/release" |
| 24 | ) |
| 25 | |
| 26 | func TestClaimServingAssemblyWithoutMoves(t *testing.T) { |
| 27 | fixtureWithoutRootMigrationCensus(t) |
| 28 | t.Setenv("CARLOS_HOST_CLAIM", "1") |
| 29 | t.Setenv("CARLOS_HOST_CLAIM_ORIGINS", "1") |
| 30 | t.Setenv("CARLOS_MOVES_PHASE1", "") |
| 31 | for _, ha := range []string{"", "1"} { |
| 32 | t.Run("region-ha="+ha, func(t *testing.T) { |
| 33 | t.Setenv("CARLOS_REGION_HA", ha) |
| 34 | store, resources, owners, _ := fix4JournalFixture(t) |
| 35 | ctx, err := claimEntryContext(t.Context(), os.Getenv, func(context.Context) (release.Store, error) { return store, nil }) |
| 36 | if err != nil { |
| 37 | t.Fatal(err) |
| 38 | } |
| 39 | actor, err := owners.Identity.Verify(ctx) |
| 40 | if err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | data := leasetest.NewVersionedBucket() |
| 44 | data.AddBucket(actor.Bucket) |
| 45 | rt := registry.Route{Host: filesHost, Kind: "instance", Hibernate: true, DBPath: provision.NestedDBPath(resources.DataDir, filesHost), Socket: provision.NestedSocket(filesHost)} |
| 46 | dir := filepath.Dir(rt.DBPath) |
| 47 | if err := os.MkdirAll(dir, 0700); err != nil { |
| 48 | t.Fatal(err) |
| 49 | } |
| 50 | if err := os.WriteFile(filepath.Join(dir, filesHost+".env"), []byte("K=1\n"), 0600); err != nil { |
| 51 | t.Fatal(err) |
| 52 | } |
| 53 | source := &movesFilesSource{DataDir: resources.DataDir, Routes: testClaimRoutes{rows: map[string]registry.Route{filesHost: rt}}, newFiles: func(context.Context, string, string) (release.Store, error) { |
| 54 | return casBucketStore{b: data, bucket: actor.Bucket}, nil |
| 55 | }} |
| 56 | files, err := source.Open(ctx, store, data, actor) |
| 57 | if err != nil { |
| 58 | t.Fatal(err) |
| 59 | } |
| 60 | d5 := plan9D5Fixture(t) |
| 61 | owners.Data = data |
| 62 | owners.Serving = d5.owners.Sources |
| 63 | owners.Serving.Files = files |
| 64 | owners.Serving.Markers = plan9D5Markers{data} |
| 65 | assembly, err := OpenProductionClaimAssembly(ctx, store, resources, owners) |
| 66 | if err != nil { |
| 67 | t.Fatal(err) |
| 68 | } |
| 69 | defer assembly.Close(ctx) |
| 70 | if assembly.wakeUnavailable != nil { |
| 71 | t.Fatal(assembly.wakeUnavailable) |
| 72 | } |
| 73 | dispatch, ok := assembly.Wake.Inputs.(ClaimOwnerDispatch) |
| 74 | if !ok || dispatch.Moves != nil || dispatch.Relocation != nil || assembly.Wake.Operations != nil || assembly.Wake.ProbeReady != nil { |
| 75 | t.Fatalf("serving-only assembly: %#v", assembly.Wake) |
| 76 | } |
| 77 | if digest, err := assembly.Wake.Retries.Inputs(ctx, rt); err != nil || digest == "" { |
| 78 | t.Fatalf("digest %q: %v", digest, err) |
| 79 | } |
| 80 | o, err := hostclaim.Read(ctx, store, filesHost) |
| 81 | if err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | n := hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + filesHost + ".json", Account: "bab", App: "jam", Host: filesHost, Region: actor.Region, CreatedAt: filesCreated} |
| 85 | o, err = hostclaim.Update(ctx, store, o, hostclaim.BeginCreate(n, []byte(`{"host":"shop.example.com","account":"bab","app":"jam","created_at":"2026-10-01T00:00:00Z"}`), "create", "console", time.Now())) |
| 86 | if err != nil { |
| 87 | t.Fatal(err) |
| 88 | } |
| 89 | o, err = hostclaim.Update(ctx, store, o, hostclaim.CompleteCreate("create", actor.Box)) |
| 90 | if err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | in, err := dispatch.Select(ctx, rt, o, recoveryPolicy) |
| 94 | if err != nil || !in.Empty { |
| 95 | t.Fatalf("empty selection: %+v %v", in, err) |
| 96 | } |
| 97 | writer := hostbox.Writer{Claims: store, Box: actor.Box, Region: actor.Region, AdmissionEvidence: assembly.AdmissionEvidence} |
| 98 | admitted, err := writer.Admit(ctx, o, in.Admission, in.Baseline) |
| 99 | if err != nil { |
| 100 | t.Fatal(err) |
| 101 | } |
| 102 | work := &filesFixture{t: t} |
| 103 | restore := hostbox.ColdRestore{Generation: in.Admission.NewGeneration, Work: work, Identity: in.Identity} |
| 104 | if err := restore.InstallEmpty(ctx, rt.DBPath, func(ctx context.Context) error { return dispatch.EmptyAuthority(ctx, admitted, in) }); err != nil { |
| 105 | t.Fatal(err) |
| 106 | } |
| 107 | if err := dispatch.Materialize(ctx, admitted, in, work); err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | key := movesFilesPrefix(admitted.Claim.Holder.Location, in.Admission.NewGeneration) + "manifest.json" |
| 111 | if data.LatestID(actor.Bucket, key) == "" { |
| 112 | t.Fatal("production C8 did not commit") |
| 113 | } |
| 114 | if assembly.Wake.BeforeLegacyInstall == nil || assembly.Wake.RecoverArtifacts == nil || assembly.Wake.BeforeInstallLocked == nil || assembly.Wake.Retries.BeforeReserve == nil { |
| 115 | t.Fatal("serving retirement or capacity not wired") |
| 116 | } |
| 117 | rootKey := fmt.Sprintf("%s/g-%s/0000/0000000000000001-0000000000000001.ltx", admitted.Claim.Holder.Location, in.Admission.NewGeneration) |
| 118 | syncs := 0 |
| 119 | publication := ClaimPublication{Sync: func(context.Context) error { syncs++; data.Put(actor.Bucket, rootKey, plan9D5RootBody(t)); return nil }, Check: func(context.Context) error { return nil }} |
| 120 | receipt, err := assembly.Wake.Publication.PublishFenced(ctx, admitted, in, publication) |
| 121 | if err != nil || receipt.MarkerVersion == "" || receipt.Root.Key != rootKey || syncs != 1 { |
| 122 | t.Fatal("moves-off publication", receipt, err, syncs) |
| 123 | } |
| 124 | writer.PublicationEvidence = assembly.PublicationEvidence |
| 125 | writer.NewID = func() string { return "serving-published" } |
| 126 | published, err := writer.Published(ctx, admitted) |
| 127 | if err != nil || published.Claim.Holder.Generation != in.Admission.NewGeneration { |
| 128 | t.Fatal("moves-off publication evidence", err) |
| 129 | } |
| 130 | sockdir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "seed-ready-") |
| 131 | if err != nil { |
| 132 | t.Fatal(err) |
| 133 | } |
| 134 | defer os.RemoveAll(sockdir) |
| 135 | rt.Socket = filepath.Join(sockdir, "ready.sock") |
| 136 | listener, err := net.Listen("unix", rt.Socket) |
| 137 | if err != nil { |
| 138 | t.Fatal(err) |
| 139 | } |
| 140 | var requests atomic.Int32 |
| 141 | server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(http.StatusOK) })} |
| 142 | done := make(chan struct{}) |
| 143 | go func() { defer close(done); _ = server.Serve(listener) }() |
| 144 | defer func() { server.Close(); <-done }() |
| 145 | process, err := os.FindProcess(os.Getpid()) |
| 146 | if err != nil { |
| 147 | t.Fatal(err) |
| 148 | } |
| 149 | readyCtx, cancel := context.WithTimeout(ctx, time.Second) |
| 150 | err = assembly.Wake.Ready(readyCtx, &exec.Cmd{Process: process}, rt, in) |
| 151 | cancel() |
| 152 | if err != nil || requests.Load() != 1 { |
| 153 | t.Fatal("moves-off readiness", err, requests.Load()) |
| 154 | } |
| 155 | wrong, _ := os.FindProcess(os.Getpid() + 1000000) |
| 156 | readyCtx, cancel = context.WithTimeout(ctx, 40*time.Millisecond) |
| 157 | err = assembly.Wake.Ready(readyCtx, &exec.Cmd{Process: wrong}, rt, in) |
| 158 | cancel() |
| 159 | if err == nil || requests.Load() != 1 { |
| 160 | t.Fatal("readiness accepted another process", err, requests.Load()) |
| 161 | } |
| 162 | |
| 163 | for _, kind := range []string{hostclaim.KindMoving, hostclaim.KindLanding, hostclaim.KindReverting, hostclaim.KindRestoring, hostclaim.KindRelocating} { |
| 164 | if err := plan9D5Call(dispatch, plan9D5Kind(kind)); err == nil { |
| 165 | t.Fatalf("moves-off accepted %s", kind) |
| 166 | } |
| 167 | } |
| 168 | }) |
| 169 | } |
| 170 | } |
| 171 | |
| 172 | func TestClaimServingMissingSourcesRefusesBeforeSelection(t *testing.T) { |
| 173 | for _, field := range []string{"launch", "markers", "files"} { |
| 174 | t.Run(field, func(t *testing.T) { |
| 175 | f := plan9D5Fixture(t) |
| 176 | switch field { |
| 177 | case "launch": |
| 178 | f.owners.Sources.Launch = nil |
| 179 | case "markers": |
| 180 | f.owners.Sources.Markers = nil |
| 181 | case "files": |
| 182 | f.owners.Sources.Files = nil |
| 183 | } |
| 184 | var dependency *ClaimDependencyError |
| 185 | if err := checkClaimServingSources(f.owners.Sources); !errors.As(err, &dependency) { |
| 186 | t.Fatalf("missing %s: %v", field, err) |
| 187 | } |
| 188 | _, err := f.owners.Select(t.Context(), registry.Route{}, plan9D5Observed(hostclaim.Holder{}), recoveryPolicy) |
| 189 | if !errors.As(err, &dependency) { |
| 190 | t.Fatalf("selection with missing %s: %v", field, err) |
| 191 | } |
| 192 | if len(f.files) != 0 || f.seq != 0 { |
| 193 | t.Fatal("missing dependency had effects") |
| 194 | } |
| 195 | }) |
| 196 | } |
| 197 | } |
| 198 | |
| 199 | func TestClaimServingAssemblyFlagMatrix(t *testing.T) { |
| 200 | fixtureWithoutRootMigrationCensus(t) |
| 201 | for _, claims := range []string{"", "1"} { |
| 202 | for _, moves := range []string{"", "1"} { |
| 203 | for _, partial := range []bool{false, true} { |
| 204 | name := "claims=" + claims + "/moves=" + moves |
| 205 | if partial { |
| 206 | name += "/partial" |
| 207 | } |
| 208 | t.Run(name, func(t *testing.T) { |
| 209 | t.Setenv("CARLOS_HOST_CLAIM", claims) |
| 210 | t.Setenv("CARLOS_HOST_CLAIM_ORIGINS", "") |
| 211 | t.Setenv("CARLOS_MOVES_PHASE1", moves) |
| 212 | t.Setenv("CARLOS_REGION_HA", "1") |
| 213 | store, resources, owners, _ := fix4JournalFixture(t) |
| 214 | ctx, err := claimEntryContext(t.Context(), os.Getenv, func(context.Context) (release.Store, error) { return store, nil }) |
| 215 | if err != nil { |
| 216 | t.Fatal(err) |
| 217 | } |
| 218 | f := plan9D5Fixture(t) |
| 219 | owners.Serving = f.owners.Sources |
| 220 | var calls []string |
| 221 | owners.Moves = ClaimMovesOwners{ClaimKindOwners: plan9D5Owner{"moves", &calls}.owners(), Operations: &plan9D2Attempts{}} |
| 222 | if partial { |
| 223 | owners.Moves.PublicationEvidence = nil |
| 224 | } |
| 225 | assembly, err := OpenProductionClaimAssembly(ctx, store, resources, owners) |
| 226 | if err != nil { |
| 227 | t.Fatal(err) |
| 228 | } |
| 229 | defer assembly.Close(ctx) |
| 230 | dispatch, bound := assembly.Wake.Inputs.(ClaimOwnerDispatch) |
| 231 | if bound != (claims != "") { |
| 232 | t.Fatalf("serving binding %v, claims %q", bound, claims) |
| 233 | } |
| 234 | wantMoves := claims != "" && moves != "" && !partial |
| 235 | if bound && (dispatch.Moves != nil) != wantMoves { |
| 236 | t.Fatalf("moves binding %v, want %v", dispatch.Moves != nil, wantMoves) |
| 237 | } |
| 238 | if (assembly.Wake.Operations != nil) != wantMoves { |
| 239 | t.Fatal("operation gate changed") |
| 240 | } |
| 241 | wantMissing := claims != "" && moves != "" && partial |
| 242 | if (assembly.wakeUnavailable != nil) != wantMissing { |
| 243 | t.Fatalf("partial owner refusal %v, want %v", assembly.wakeUnavailable, wantMissing) |
| 244 | } |
| 245 | if bound && dispatch.Relocation != nil { |
| 246 | t.Fatal("region HA bypassed absent relocation owners") |
| 247 | } |
| 248 | if len(calls) != 0 || len(f.files) != 0 || f.seq != 0 { |
| 249 | t.Fatal("assembly ran a host operation") |
| 250 | } |
| 251 | }) |
| 252 | } |
| 253 | } |
| 254 | } |
| 255 | } |
| 256 | |
amadan