| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/sha256" |
| 6 | "database/sql" |
| 7 | "encoding/hex" |
| 8 | "encoding/json" |
| 9 | "errors" |
| 10 | "fmt" |
| 11 | "io" |
| 12 | "log/slog" |
| 13 | "net/http/httptest" |
| 14 | "os" |
| 15 | "path/filepath" |
| 16 | "strconv" |
| 17 | "strings" |
| 18 | "testing" |
| 19 | "time" |
| 20 | |
| 21 | "github.com/benbjohnson/litestream" |
| 22 | lsfile "github.com/benbjohnson/litestream/file" |
| 23 | "github.com/carlosframework/platform/internal/genselect" |
| 24 | "github.com/carlosframework/platform/internal/hostbox" |
| 25 | "github.com/carlosframework/platform/internal/hostclaim" |
| 26 | execlease "github.com/carlosframework/platform/internal/hostexec" |
| 27 | "github.com/carlosframework/platform/internal/hostfiles" |
| 28 | "github.com/carlosframework/platform/internal/hostguard" |
| 29 | "github.com/carlosframework/platform/internal/hostrelocate" |
| 30 | "github.com/carlosframework/platform/internal/lease" |
| 31 | "github.com/carlosframework/platform/internal/lease/leasetest" |
| 32 | "github.com/carlosframework/platform/internal/moves" |
| 33 | "github.com/carlosframework/platform/internal/release" |
| 34 | "github.com/superfly/ltx" |
| 35 | ) |
| 36 | |
| 37 | const buildHost = "shop.example" |
| 38 | |
| 39 | // buildFixture is a relocation target at installing on an empty box: the |
| 40 | // claim journals a pin of a real litestream replica sealed into a |
| 41 | // versioned bucket (served over a loopback S3 endpoint, so the replica |
| 42 | // lease is lease.Acquire's own), a retained files manifest with one entry, |
| 43 | // an empty guard directory, and main's Local recorded. |
| 44 | type buildFixture struct { |
| 45 | t *testing.T |
| 46 | claims *release.DirStore |
| 47 | files *hookedStore |
| 48 | bucket *leasetest.VersionedBucket |
| 49 | versions *hookedVersions |
| 50 | a ClaimAssembly |
| 51 | cfg RelocationAssemblyConfig |
| 52 | obs hostclaim.Observed |
| 53 | locals []hostbox.Selection |
| 54 | localErr error |
| 55 | // localHook, when set, is asked on every Local check that localErr |
| 56 | // passes: a test loses the lease at one write group. |
| 57 | localHook func(hostbox.Selection) error |
| 58 | verifier PreparedFromInstallManifest |
| 59 | } |
| 60 | |
| 61 | // hookedStore and hookedVersions run after, once a write has landed, so a |
| 62 | // test can move the build's authority between two durable writes. |
| 63 | type hookedStore struct { |
| 64 | release.Store |
| 65 | afterPut func(key string) |
| 66 | } |
| 67 | |
| 68 | func (s *hookedStore) Put(ctx context.Context, key string, body []byte, etag string) (string, error) { |
| 69 | out, err := s.Store.Put(ctx, key, body, etag) |
| 70 | if err == nil && s.afterPut != nil { |
| 71 | s.afterPut(key) |
| 72 | } |
| 73 | return out, err |
| 74 | } |
| 75 | |
| 76 | type hookedVersions struct { |
| 77 | lease.VersionedStore |
| 78 | afterPut func(key string) |
| 79 | } |
| 80 | |
| 81 | func (s *hookedVersions) PutIfAbsent(ctx context.Context, bucket, key string, body []byte) (string, error) { |
| 82 | out, err := s.VersionedStore.PutIfAbsent(ctx, bucket, key, body) |
| 83 | if err == nil && s.afterPut != nil { |
| 84 | s.afterPut(key) |
| 85 | } |
| 86 | return out, err |
| 87 | } |
| 88 | |
| 89 | // buildOptions shape a buildFixture: a pin with or without a database, the |
| 90 | // data root (a fresh one with a hostguard directory when empty), and the |
| 91 | // pinned manifest's entries and platform items, each stored as an object |
| 92 | // under the source generation's prefix. NoFiles pins a host that never |
| 93 | // published files instead (no digest, no retained copy: the pinner's |
| 94 | // evidenced empty files set), and Files and Platform are ignored. |
| 95 | type buildOptions struct { |
| 96 | DB bool |
| 97 | Data string |
| 98 | Files map[string]string |
| 99 | Platform map[string]string |
| 100 | NoFiles bool |
| 101 | } |
| 102 | |
| 103 | func newBuildFixture(t *testing.T, withDB bool) *buildFixture { |
| 104 | return newBuildFixtureWith(t, buildOptions{DB: withDB, Files: map[string]string{"public/hello.txt": "hello"}}) |
| 105 | } |
| 106 | |
| 107 | func newBuildFixtureWith(t *testing.T, o buildOptions) *buildFixture { |
| 108 | t.Helper() |
| 109 | ctx := t.Context() |
| 110 | withDB := o.DB |
| 111 | f := &buildFixture{t: t, claims: release.NewDirStore(t.TempDir()), bucket: leasetest.NewVersionedBucket()} |
| 112 | f.bucket.AddBucket("data") |
| 113 | // P's files manifests and the pinned objects live in the versioned data |
| 114 | // bucket, as production's files store and versioned client are one bucket. |
| 115 | f.files = &hookedStore{Store: bucketStore{b: f.bucket, bucket: "data"}} |
| 116 | f.versions = &hookedVersions{VersionedStore: f.bucket} |
| 117 | handler, _ := leasetest.NewS3Server(f.bucket, "data") |
| 118 | srv := httptest.NewServer(handler) // 127.0.0.1 only |
| 119 | t.Cleanup(srv.Close) |
| 120 | t.Setenv("CARLOS_S3_ENDPOINT", srv.URL) |
| 121 | t.Setenv("AWS_ACCESS_KEY_ID", "test-key") |
| 122 | t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret") |
| 123 | t.Setenv("AWS_SESSION_TOKEN", "") |
| 124 | t.Setenv("AWS_PROFILE", "") |
| 125 | t.Setenv("AWS_REGION", "") |
| 126 | |
| 127 | created := time.Date(2026, 10, 7, 1, 0, 0, 0, time.UTC) |
| 128 | enrollment := strings.Repeat("a", 32) |
| 129 | nh := hostclaim.NewHolder{Family: "instance", Key: "apps/acct/app/instances/" + buildHost + ".json", Account: "acct", App: "app", Host: buildHost, Region: "ie", RecordRegion: "ie", CreatedAt: created} |
| 130 | body := []byte(`{"host":"` + buildHost + `","account":"acct","app":"app","region":"ie","created_at":"2026-10-07T01:00:00Z"}`) |
| 131 | obs, err := hostclaim.Read(ctx, f.claims, buildHost) |
| 132 | if err != nil { |
| 133 | t.Fatal(err) |
| 134 | } |
| 135 | for _, tr := range []func(*hostclaim.Claim) error{ |
| 136 | hostclaim.BeginCreateOn(nh, body, "c1", "console", created, hostclaim.ExecutorSelection{Box: "src", Enrollment: enrollment}), |
| 137 | hostclaim.CompleteCreate("c1", "src"), |
| 138 | hostclaim.AdmitWriter(hostclaim.Admission{Box: "src", Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true}), |
| 139 | } { |
| 140 | if obs, err = hostclaim.Update(ctx, f.claims, obs, tr); err != nil { |
| 141 | t.Fatal(err) |
| 142 | } |
| 143 | } |
| 144 | location := obs.Claim.Holder.Location |
| 145 | const op = "rel-1" |
| 146 | |
| 147 | pin := hostrelocate.PinObject{V: 1, Host: buildHost, Op: op, ClaimGen: obs.Claim.Gen, PinnedAt: created, Release: "v1"} |
| 148 | if withDB { |
| 149 | files, n := sealedReplica(t, f.bucket, moves.GenerationPrefix(location, "g1")) |
| 150 | pin.DB = &genselect.Plan{Generation: "g1", Files: files, MaxTXID: n, MaxLevel0TXID: n} |
| 151 | } else { |
| 152 | pin.Empty = "session-0" |
| 153 | } |
| 154 | srcPrefix := location + "/files/g-g1/" |
| 155 | pinned := hostfiles.Manifest{V: hostfiles.ManifestVersion, Seq: 5, Platform: map[string]hostfiles.Entry{}, Entries: map[string]hostfiles.Entry{}} |
| 156 | object := func(body string, mode uint32) hostfiles.Entry { |
| 157 | sum := sha256.Sum256([]byte(body)) |
| 158 | sha := hex.EncodeToString(sum[:]) |
| 159 | key := srcPrefix + "objects/" + fmt.Sprintf("%016d-%s", 5, sha) |
| 160 | if _, err := f.files.Store.Put(ctx, key, []byte(body), ""); err != nil { |
| 161 | t.Fatal(err) |
| 162 | } |
| 163 | return hostfiles.Entry{Object: key, SHA256: sha, Size: int64(len(body)), Mode: mode, Seq: 5} |
| 164 | } |
| 165 | for rel, body := range o.Files { |
| 166 | pinned.Entries[rel] = object(body, 0o644) |
| 167 | } |
| 168 | for name, body := range o.Platform { |
| 169 | pinned.Platform[name] = object(body, 0o600) |
| 170 | } |
| 171 | pinnedRaw, err := json.Marshal(pinned) |
| 172 | if err != nil { |
| 173 | t.Fatal(err) |
| 174 | } |
| 175 | pinnedSum := sha256.Sum256(pinnedRaw) |
| 176 | retained := hostrelocate.RetainedManifestKey(buildHost, op, hex.EncodeToString(pinnedSum[:])) |
| 177 | if _, err := f.claims.Put(ctx, retained, pinnedRaw, ""); err != nil { |
| 178 | t.Fatal(err) |
| 179 | } |
| 180 | pin.Files = hostrelocate.FilesPin{Prefix: srcPrefix, Seq: 5, SHA256: hex.EncodeToString(pinnedSum[:]), Origins: []string{srcPrefix}, Retained: retained} |
| 181 | filesManifest, filesSeq := retained, int64(5) |
| 182 | if o.NoFiles { |
| 183 | pin.Files = hostrelocate.FilesPin{Prefix: srcPrefix, Origins: []string{srcPrefix}} |
| 184 | filesManifest, filesSeq = hostfiles.ManifestKey(srcPrefix), 0 |
| 185 | } |
| 186 | pinRaw, err := json.Marshal(pin) |
| 187 | if err != nil { |
| 188 | t.Fatal(err) |
| 189 | } |
| 190 | pinKey := hostrelocate.PinKey(buildHost, op, obs.Claim.Gen) |
| 191 | if _, err := f.claims.Put(ctx, pinKey, pinRaw, ""); err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | pinSum := sha256.Sum256(pinRaw) |
| 195 | ref := hostclaim.RecoveryPin{Key: pinKey, SHA256: hex.EncodeToString(pinSum[:]), Generation: "g1", FilesManifest: filesManifest, FilesSeq: filesSeq, PinnedAt: created, Release: "v1", Empty: pin.DB == nil} |
| 196 | if pin.DB != nil { |
| 197 | ref.MaxTXID = pin.DB.MaxTXID |
| 198 | } |
| 199 | refRaw, err := json.Marshal(ref) |
| 200 | if err != nil { |
| 201 | t.Fatal(err) |
| 202 | } |
| 203 | // The installing CAS: this box executes the op, the holder's stamp is |
| 204 | // the install stamp, P is named (hostclaim's install()). |
| 205 | if obs, err = hostclaim.Update(ctx, f.claims, obs, func(c *hostclaim.Claim) error { |
| 206 | stamp := c.Gen + 1 |
| 207 | c.Holder.Stamp = stamp |
| 208 | c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: op, Phase: hostclaim.PhaseInstalling, Exec: "dst", Participants: []string{"src", "dst"}, |
| 209 | Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": "dst"}, |
| 210 | Journal: map[string]string{"pin": string(refRaw), "prepared": "p1", "install_stamp": strconv.FormatUint(stamp, 10), "install_release": "v1"}} |
| 211 | return nil |
| 212 | }); err != nil { |
| 213 | t.Fatal(err) |
| 214 | } |
| 215 | f.obs = obs |
| 216 | |
| 217 | data := o.Data |
| 218 | if data == "" { |
| 219 | data = t.TempDir() |
| 220 | } |
| 221 | if err := os.MkdirAll(filepath.Join(data, "hostguard"), 0o770); err != nil { |
| 222 | t.Fatal(err) |
| 223 | } |
| 224 | f.a = ClaimAssembly{Claims: f.claims, Guard: hostguard.Guard{DataDir: data, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: f.claims}}} |
| 225 | f.cfg = RelocationAssemblyConfig{Data: f.files, Versions: f.versions, Bucket: "data", Region: "ie", LeaseRegion: "eu-west-1", DataDir: data, |
| 226 | WorkDir: filepath.Join(data, "relocate", "build"), Now: func() time.Time { return created.Add(time.Hour) }, |
| 227 | Source: hostbox.WorkSource{Box: "dst", Region: "ie", Local: func(_ context.Context, c *hostclaim.Claim, s hostbox.Selection) error { |
| 228 | f.locals = append(f.locals, s) |
| 229 | if f.localErr == nil && f.localHook != nil { |
| 230 | return f.localHook(s) |
| 231 | } |
| 232 | return f.localErr |
| 233 | }}} |
| 234 | f.verifier = PreparedFromInstallManifest{Claims: f.claims, Files: f.files, Data: f.bucket, Bucket: "data"} |
| 235 | return f |
| 236 | } |
| 237 | |
| 238 | // sealedReplica writes a real SQLite database through litestream's file |
| 239 | // replica and seals every LTX file into the bucket under prefix, keyed as |
| 240 | // genselect lists a generation (<prefix><level>/<min>-<max>.ltx, prefix |
| 241 | // moves.GenerationPrefix). |
| 242 | func sealedReplica(t *testing.T, b *leasetest.VersionedBucket, prefix string) ([]genselect.File, uint64) { |
| 243 | t.Helper() |
| 244 | ctx := t.Context() |
| 245 | dir := t.TempDir() |
| 246 | path := filepath.Join(dir, "src.db") |
| 247 | write := func(n int) { |
| 248 | d, err := sql.Open("sqlite", path) |
| 249 | if err != nil { |
| 250 | t.Fatal(err) |
| 251 | } |
| 252 | defer d.Close() |
| 253 | if _, err := d.Exec(`CREATE TABLE IF NOT EXISTS t (id INTEGER PRIMARY KEY, body TEXT)`); err != nil { |
| 254 | t.Fatal(err) |
| 255 | } |
| 256 | for range n { |
| 257 | if _, err := d.Exec(`INSERT INTO t (body) VALUES (?)`, strings.Repeat("x", 100)); err != nil { |
| 258 | t.Fatal(err) |
| 259 | } |
| 260 | } |
| 261 | } |
| 262 | write(1) |
| 263 | client := lsfile.NewReplicaClient(filepath.Join(dir, "replica")) |
| 264 | db := litestream.NewDB(path) |
| 265 | db.Logger = slog.New(slog.NewTextHandler(io.Discard, nil)) |
| 266 | db.SetMetaPath(filepath.Join(dir, "meta")) |
| 267 | db.MonitorInterval = time.Hour |
| 268 | db.Replica = litestream.NewReplicaWithClient(db, client) |
| 269 | db.Replica.SyncInterval = time.Hour |
| 270 | if err := db.Open(); err != nil { |
| 271 | t.Fatal(err) |
| 272 | } |
| 273 | for range 2 { |
| 274 | if err := db.SyncAndWait(ctx); err != nil { |
| 275 | t.Fatal(err) |
| 276 | } |
| 277 | write(2) |
| 278 | } |
| 279 | if err := db.SyncAndWait(ctx); err != nil { |
| 280 | t.Fatal(err) |
| 281 | } |
| 282 | if err := db.Close(ctx); err != nil { |
| 283 | t.Fatal(err) |
| 284 | } |
| 285 | var out []genselect.File |
| 286 | var n uint64 |
| 287 | for level := 0; level <= litestream.SnapshotLevel; level++ { |
| 288 | itr, err := client.LTXFiles(ctx, level, 0, false) |
| 289 | if err != nil { |
| 290 | t.Fatal(err) |
| 291 | } |
| 292 | infos, err := ltx.SliceFileIterator(itr) |
| 293 | if err != nil { |
| 294 | t.Fatal(err) |
| 295 | } |
| 296 | for _, info := range infos { |
| 297 | rc, err := client.OpenLTXFile(ctx, info.Level, info.MinTXID, info.MaxTXID, 0, 0) |
| 298 | if err != nil { |
| 299 | t.Fatal(err) |
| 300 | } |
| 301 | body, err := io.ReadAll(rc) |
| 302 | rc.Close() |
| 303 | if err != nil { |
| 304 | t.Fatal(err) |
| 305 | } |
| 306 | key := prefix + fmt.Sprintf("%04x/", level) + ltx.FormatFilename(info.MinTXID, info.MaxTXID) |
| 307 | v := b.Put("data", key, body) |
| 308 | out = append(out, genselect.File{Key: key, VersionID: v, Level: level, MinTXID: uint64(info.MinTXID), MaxTXID: uint64(info.MaxTXID), Size: int64(len(body))}) |
| 309 | if level == 0 && uint64(info.MaxTXID) > n { |
| 310 | n = uint64(info.MaxTXID) |
| 311 | } |
| 312 | } |
| 313 | } |
| 314 | if n == 0 { |
| 315 | t.Fatal("the source replica has no level-0 files") |
| 316 | } |
| 317 | return out, n |
| 318 | } |
| 319 | |
| 320 | func (f *buildFixture) build() error { |
| 321 | return f.a.relocationBuild(f.cfg)(f.t.Context(), f.obs) |
| 322 | } |
| 323 | |
| 324 | func (f *buildFixture) prefix() string { |
| 325 | return moves.GenerationPrefix(f.obs.Claim.Holder.Location, "p1") |
| 326 | } |
| 327 | |
| 328 | func (f *buildFixture) installManifestExists() bool { |
| 329 | _, ok := f.bucket.Current("data", moves.InstallManifestKey(f.prefix())) |
| 330 | return ok |
| 331 | } |
| 332 | |
| 333 | // moveExec moves the op to another executor, as a replaced target sees it. |
| 334 | func (f *buildFixture) moveExec() { |
| 335 | f.t.Helper() |
| 336 | o, err := hostclaim.Read(context.Background(), f.claims, buildHost) |
| 337 | if err != nil { |
| 338 | f.t.Fatal(err) |
| 339 | } |
| 340 | if _, err := hostclaim.Update(context.Background(), f.claims, o, func(c *hostclaim.Claim) error { |
| 341 | c.Op.Exec, c.Op.Intent["target_box"] = "other", "other" |
| 342 | c.Op.Participants = []string{"src", "other"} |
| 343 | return nil |
| 344 | }); err != nil { |
| 345 | f.t.Fatal(err) |
| 346 | } |
| 347 | } |
| 348 | |
| 349 | func (f *buildFixture) retireStamp() { |
| 350 | f.t.Helper() |
| 351 | if err := f.a.Guard.RetireInstallation(context.Background(), buildHost, func(*hostclaim.Claim) error { return nil }); err != nil { |
| 352 | f.t.Fatal(err) |
| 353 | } |
| 354 | } |
| 355 | |
| 356 | // Prepared is done only when P's files manifest and, for a non-empty pin, |
| 357 | // its install manifest both verify; an unconfigured verifier reports |
| 358 | // nothing built. Done carries what ToInstalled requires: P's manifest at |
| 359 | // sequence 0 built from the pinned sequence, and the install manifest at |
| 360 | // the pin's N. |
| 361 | func TestPreparedFromInstallManifestNeedsBothManifests(t *testing.T) { |
| 362 | if _, done, err := (PreparedFromInstallManifest{}).Verify(t.Context(), hostclaim.Observed{Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{}}}); done || err == nil { |
| 363 | t.Fatalf("an unconfigured verifier reported P built: %v", err) |
| 364 | } |
| 365 | f := newBuildFixture(t, true) |
| 366 | if _, done, err := f.verifier.Verify(t.Context(), f.obs); done || err != nil { |
| 367 | t.Fatalf("before the build: done=%v %v", done, err) |
| 368 | } |
| 369 | if err := f.build(); err != nil { |
| 370 | t.Fatal(err) |
| 371 | } |
| 372 | a, done, err := f.verifier.Verify(t.Context(), f.obs) |
| 373 | if err != nil || !done { |
| 374 | t.Fatalf("after the build: done=%v %v", done, err) |
| 375 | } |
| 376 | var ref hostclaim.RecoveryPin |
| 377 | _ = json.Unmarshal([]byte(f.obs.Claim.Op.Journal["pin"]), &ref) |
| 378 | if a.Generation != "p1" || a.FilesSeq != 0 || a.SourceFilesSeq != 5 || len(a.FilesManifestSHA256) != 64 || |
| 379 | a.InstallManifest != moves.InstallManifestKey(f.prefix()) || len(a.SHA256) != 64 || a.SnapshotTXID != ref.MaxTXID { |
| 380 | t.Fatalf("authority: %+v", a) |
| 381 | } |
| 382 | // An empty bootstrap needs only P's files manifest. |
| 383 | empty := newBuildFixture(t, false) |
| 384 | if err := empty.build(); err != nil { |
| 385 | t.Fatal(err) |
| 386 | } |
| 387 | if a, done, err := empty.verifier.Verify(t.Context(), empty.obs); err != nil || !done || a.InstallManifest != "" || a.SnapshotTXID != 0 { |
| 388 | t.Fatalf("empty bootstrap: %+v done=%v %v", a, done, err) |
| 389 | } |
| 390 | if empty.installManifestExists() { |
| 391 | t.Fatal("an empty pin built a snapshot") |
| 392 | } |
| 393 | } |
| 394 | |
| 395 | // A crash after the files manifest and before the install manifest, and a |
| 396 | // store holding the install manifest without P's files manifest, each |
| 397 | // leave P unverified, never an error that would stop the worker. |
| 398 | func TestPreparedRefusesAPartialBuild(t *testing.T) { |
| 399 | t.Run("files manifest only", func(t *testing.T) { |
| 400 | f := newBuildFixture(t, true) |
| 401 | // The process dies after P's files manifest: the next authority |
| 402 | // check refuses, as a crash would stop it. |
| 403 | f.files.afterPut = func(string) { f.localErr = errors.New("crashed") } |
| 404 | if err := f.build(); err == nil { |
| 405 | t.Fatal("the stopped build succeeded") |
| 406 | } |
| 407 | if _, _, err := f.files.Get(t.Context(), hostfiles.ManifestKey(f.obs.Claim.Holder.Location+"/files/g-p1/")); err != nil { |
| 408 | t.Fatalf("P's files manifest was not written first: %v", err) |
| 409 | } |
| 410 | if _, done, err := f.verifier.Verify(t.Context(), f.obs); done || err != nil { |
| 411 | t.Fatalf("done=%v err=%v", done, err) |
| 412 | } |
| 413 | }) |
| 414 | t.Run("install manifest only", func(t *testing.T) { |
| 415 | f := newBuildFixture(t, true) |
| 416 | if err := f.build(); err != nil { |
| 417 | t.Fatal(err) |
| 418 | } |
| 419 | v := f.verifier |
| 420 | v.Files = release.NewDirStore(t.TempDir()) // P's files manifest is not there |
| 421 | if _, done, err := v.Verify(t.Context(), f.obs); done || err != nil { |
| 422 | t.Fatalf("done=%v err=%v", done, err) |
| 423 | } |
| 424 | }) |
| 425 | t.Run("a files manifest that is not the pinned entries", func(t *testing.T) { |
| 426 | f := newBuildFixture(t, true) |
| 427 | bad := hostfiles.Manifest{V: hostfiles.ManifestVersion, Entries: map[string]hostfiles.Entry{}, Platform: map[string]hostfiles.Entry{}} |
| 428 | raw, _ := json.Marshal(bad) |
| 429 | if _, err := f.files.Store.Put(t.Context(), hostfiles.ManifestKey(f.obs.Claim.Holder.Location+"/files/g-p1/"), raw, ""); err != nil { |
| 430 | t.Fatal(err) |
| 431 | } |
| 432 | if _, done, err := f.verifier.Verify(t.Context(), f.obs); done || !errors.Is(err, hostrelocate.ErrRefused) { |
| 433 | t.Fatalf("done=%v err=%v", done, err) |
| 434 | } |
| 435 | }) |
| 436 | } |
| 437 | |
| 438 | // A build whose authority moved under it stops before the next durable |
| 439 | // write: after P's files manifest the executor moved; after the snapshot |
| 440 | // PUT the executor moved, or the guard retired the stamp. No install |
| 441 | // manifest is written for a box that no longer builds. |
| 442 | func TestRelocationBuildStopsWhenAuthorityChanges(t *testing.T) { |
| 443 | for _, tc := range []struct { |
| 444 | name string |
| 445 | hook func(f *buildFixture) |
| 446 | after string |
| 447 | }{ |
| 448 | {"after the files manifest", func(f *buildFixture) { f.moveExec() }, "files"}, |
| 449 | {"after the snapshot PUT", func(f *buildFixture) { f.moveExec() }, "snapshot"}, |
| 450 | {"after the stamp is retired", func(f *buildFixture) { f.retireStamp() }, "snapshot"}, |
| 451 | } { |
| 452 | t.Run(tc.name, func(t *testing.T) { |
| 453 | f := newBuildFixture(t, true) |
| 454 | fired := false |
| 455 | fire := func(string) { |
| 456 | if !fired { |
| 457 | fired = true |
| 458 | tc.hook(f) |
| 459 | } |
| 460 | } |
| 461 | if tc.after == "files" { |
| 462 | f.files.afterPut = fire |
| 463 | } else { |
| 464 | f.versions.afterPut = func(key string) { |
| 465 | if strings.HasPrefix(key, f.prefix()) && strings.HasSuffix(key, ".ltx") { |
| 466 | fire(key) |
| 467 | } |
| 468 | } |
| 469 | } |
| 470 | if err := f.build(); !errors.Is(err, hostrelocate.ErrRefused) || !fired { |
| 471 | t.Fatalf("the build went on after its authority changed: %v (fired %v)", err, fired) |
| 472 | } |
| 473 | if f.installManifestExists() { |
| 474 | t.Fatal("the install manifest was written for a box that no longer builds") |
| 475 | } |
| 476 | }) |
| 477 | } |
| 478 | } |
| 479 | |
| 480 | // Ruling 59: the replica lease, the claim and the guard all stay valid; |
| 481 | // only main's Local refuses (the execution lease lost), and the next write |
| 482 | // is refused with its error. |
| 483 | func TestRelocationBuildStopsWhenOnlyTheExecutionLeaseIsLost(t *testing.T) { |
| 484 | f := newBuildFixture(t, true) |
| 485 | f.versions.afterPut = func(key string) { |
| 486 | if strings.HasSuffix(key, ".ltx") { |
| 487 | f.localErr = execlease.ErrLost |
| 488 | } |
| 489 | } |
| 490 | if err := f.build(); !errors.Is(err, execlease.ErrLost) { |
| 491 | t.Fatalf("the build went on without its execution lease: %v", err) |
| 492 | } |
| 493 | if f.installManifestExists() { |
| 494 | t.Fatal("the install manifest was written after the execution lease was lost") |
| 495 | } |
| 496 | if c, err := hostguard.Capture(f.a.Guard.DataDir, buildHost); err != nil || c.Stamp != f.obs.Claim.Holder.Stamp { |
| 497 | t.Fatalf("the guard moved: %+v %v", c, err) |
| 498 | } |
| 499 | } |
| 500 | |
| 501 | // A replay from installing (a crash after the build, before ToInstalled) |
| 502 | // is a success that changes nothing: the same files manifest bytes, the |
| 503 | // same install manifest version. |
| 504 | func TestRelocationBuildReplaysToTheSameBytes(t *testing.T) { |
| 505 | f := newBuildFixture(t, true) |
| 506 | if err := f.build(); err != nil { |
| 507 | t.Fatal(err) |
| 508 | } |
| 509 | key := moves.InstallManifestKey(f.prefix()) |
| 510 | first := f.bucket.LatestID("data", key) |
| 511 | filesKey := hostfiles.ManifestKey(f.obs.Claim.Holder.Location + "/files/g-p1/") |
| 512 | before, _, err := f.files.Get(t.Context(), filesKey) |
| 513 | if err != nil { |
| 514 | t.Fatal(err) |
| 515 | } |
| 516 | if err := f.build(); err != nil { |
| 517 | t.Fatalf("replay: %v", err) |
| 518 | } |
| 519 | after, _, err := f.files.Get(t.Context(), filesKey) |
| 520 | if err != nil || string(after) != string(before) || f.bucket.LatestID("data", key) != first { |
| 521 | t.Fatalf("the replay rewrote P: %v", err) |
| 522 | } |
| 523 | if _, done, err := f.verifier.Verify(t.Context(), f.obs); !done || err != nil { |
| 524 | t.Fatalf("after the replay: done=%v %v", done, err) |
| 525 | } |
| 526 | } |
| 527 | |
| 528 | // Finding 2, ruling 43: the build activates the target's guard at |
| 529 | // install_stamp from an empty box, under main's Local, before Capture; |
| 530 | // never for another executor, never past a refusing Local, never at |
| 531 | // another phase, never a retired stamp. The build's host-named scratch is |
| 532 | // gone once P is built. |
| 533 | func TestRelocationBuildActivatesTheGuardFromAnEmptyTarget(t *testing.T) { |
| 534 | t.Run("from an empty box", func(t *testing.T) { |
| 535 | f := newBuildFixture(t, true) |
| 536 | if err := f.build(); err != nil { |
| 537 | t.Fatal(err) |
| 538 | } |
| 539 | c, err := hostguard.Capture(f.a.Guard.DataDir, buildHost) |
| 540 | if err != nil || c.Incarnation != f.obs.Claim.Holder.Incarnation || c.Stamp != f.obs.Claim.Holder.Stamp { |
| 541 | t.Fatalf("capture %+v %v", c, err) |
| 542 | } |
| 543 | if len(f.locals) == 0 || f.locals[0].Action != relocationActionInstall || f.locals[0].Captured.Stamp != f.obs.Claim.Holder.Stamp || f.locals[0].Box != "dst" { |
| 544 | t.Fatalf("main's Local was not asked for the install: %+v", f.locals) |
| 545 | } |
| 546 | if _, err := os.Stat(filepath.Join(f.cfg.WorkDir, "rel-1")); !errors.Is(err, os.ErrNotExist) { |
| 547 | t.Fatalf("the build's scratch survived: %v", err) |
| 548 | } |
| 549 | }) |
| 550 | for _, tc := range []struct { |
| 551 | name string |
| 552 | change func(f *buildFixture) |
| 553 | want error |
| 554 | }{ |
| 555 | {"another executor", func(f *buildFixture) { |
| 556 | f.moveExec() |
| 557 | f.obs, _ = hostclaim.Read(context.Background(), f.claims, buildHost) |
| 558 | }, hostrelocate.ErrRefused}, |
| 559 | {"main's Local refuses", func(f *buildFixture) { f.localErr = execlease.ErrLost }, execlease.ErrLost}, |
| 560 | {"installed, not installing", func(f *buildFixture) { |
| 561 | o, _ := hostclaim.Read(context.Background(), f.claims, buildHost) |
| 562 | f.obs, _ = hostclaim.Update(context.Background(), f.claims, o, func(c *hostclaim.Claim) error { c.Op.Phase = hostclaim.PhaseInstalled; return nil }) |
| 563 | }, hostrelocate.ErrRefused}, |
| 564 | } { |
| 565 | t.Run(tc.name, func(t *testing.T) { |
| 566 | f := newBuildFixture(t, true) |
| 567 | tc.change(f) |
| 568 | before := f.bucket.Snapshot() |
| 569 | if err := f.build(); !errors.Is(err, tc.want) { |
| 570 | t.Fatalf("got %v, want %v", err, tc.want) |
| 571 | } |
| 572 | if entries, _ := os.ReadDir(filepath.Join(f.a.Guard.DataDir, "hostguard")); len(entries) != 0 { |
| 573 | t.Fatalf("the guard was written: %v", entries) |
| 574 | } |
| 575 | if f.bucket.Snapshot() != before { |
| 576 | t.Fatal("the bucket was written") |
| 577 | } |
| 578 | }) |
| 579 | } |
| 580 | t.Run("a retired stamp", func(t *testing.T) { |
| 581 | f := newBuildFixture(t, true) |
| 582 | if _, err := f.a.relocationActivate(f.cfg)(t.Context(), f.obs); err != nil { |
| 583 | t.Fatal(err) |
| 584 | } |
| 585 | f.retireStamp() |
| 586 | before := f.bucket.Snapshot() |
| 587 | if err := f.build(); !errors.Is(err, hostguard.ErrRetiredStamp) { |
| 588 | t.Fatalf("a retired stamp was reactivated: %v", err) |
| 589 | } |
| 590 | if f.bucket.Snapshot() != before { |
| 591 | t.Fatal("the bucket was written") |
| 592 | } |
| 593 | }) |
| 594 | } |
| 595 | |
| 596 | // A pin whose files digest names no retained copy is refused, never read |
| 597 | // as a host that published no files (absence is never evidence). The |
| 598 | // pinned manifest here is itself empty, so only that check tells the two |
| 599 | // apart. |
| 600 | func TestPreparedRefusesAFilesDigestWithoutItsRetainedCopy(t *testing.T) { |
| 601 | ctx := t.Context() |
| 602 | f := newBuildFixtureWith(t, buildOptions{}) |
| 603 | if err := f.build(); err != nil { |
| 604 | t.Fatal(err) |
| 605 | } |
| 606 | if _, done, err := f.verifier.Verify(ctx, f.obs); !done || err != nil { |
| 607 | t.Fatalf("the built empty pin: done=%v %v", done, err) |
| 608 | } |
| 609 | var ref hostclaim.RecoveryPin |
| 610 | if err := json.Unmarshal([]byte(f.obs.Claim.Op.Journal["pin"]), &ref); err != nil { |
| 611 | t.Fatal(err) |
| 612 | } |
| 613 | pin, err := hostrelocate.ReadPin(ctx, f.claims, ref) |
| 614 | if err != nil { |
| 615 | t.Fatal(err) |
| 616 | } |
| 617 | if pin.Files.SHA256 == "" || pin.Files.Retained == "" { |
| 618 | t.Fatalf("the fixture's pin has no retained manifest: %+v", pin.Files) |
| 619 | } |
| 620 | pin.Files.Retained = "" |
| 621 | raw, err := json.Marshal(pin) |
| 622 | if err != nil { |
| 623 | t.Fatal(err) |
| 624 | } |
| 625 | ref.Key += ".forged" |
| 626 | if _, err := f.claims.Put(ctx, ref.Key, raw, ""); err != nil { |
| 627 | t.Fatal(err) |
| 628 | } |
| 629 | sum := sha256.Sum256(raw) |
| 630 | ref.SHA256 = hex.EncodeToString(sum[:]) |
| 631 | refRaw, err := json.Marshal(ref) |
| 632 | if err != nil { |
| 633 | t.Fatal(err) |
| 634 | } |
| 635 | c := *f.obs.Claim |
| 636 | c.Op.Journal = map[string]string{} |
| 637 | for k, v := range f.obs.Claim.Op.Journal { |
| 638 | c.Op.Journal[k] = v |
| 639 | } |
| 640 | c.Op.Journal["pin"] = string(refRaw) |
| 641 | obs := f.obs |
| 642 | obs.Claim = &c |
| 643 | if _, done, err := f.verifier.Verify(ctx, obs); done || !errors.Is(err, hostrelocate.ErrRefused) { |
| 644 | t.Fatalf("done=%v err=%v", done, err) |
| 645 | } |
| 646 | } |
| 647 | |
| 648 | // staleObs is the build's observation with change applied to a copy of its |
| 649 | // claim, as a worker holding an older read sees it; the stored claim is |
| 650 | // unchanged. |
| 651 | func (f *buildFixture) staleObs(change func(*hostclaim.Claim)) hostclaim.Observed { |
| 652 | c := *f.obs.Claim |
| 653 | h := *c.Holder |
| 654 | c.Holder = &h |
| 655 | c.Op.Journal = map[string]string{} |
| 656 | for k, v := range f.obs.Claim.Op.Journal { |
| 657 | c.Op.Journal[k] = v |
| 658 | } |
| 659 | change(&c) |
| 660 | o := f.obs |
| 661 | o.Claim = &c |
| 662 | return o |
| 663 | } |
| 664 | |
| 665 | // Rulings 43 and 52: the guard is activated, and P built, only for the |
| 666 | // installation the installing CAS issued. A stored claim whose holder stamp |
| 667 | // is not the install stamp, that lost this box, that is stuck or whose P |
| 668 | // was opened writes no guard record and nothing in the bucket. An |
| 669 | // observation older than the claim, at another install stamp, of another |
| 670 | // incarnation, or naming another P or pin, writes nothing in the bucket. |
| 671 | func TestRelocationBuildRefusesAnInstallationTheClaimDidNotIssue(t *testing.T) { |
| 672 | for _, tc := range []struct { |
| 673 | name string |
| 674 | stored func(*hostclaim.Claim) // a change to the stored claim |
| 675 | stale func(*buildFixture, *hostclaim.Claim) // a change to the observation only |
| 676 | want error |
| 677 | }{ |
| 678 | {name: "a holder stamp that is not the install stamp", stored: func(c *hostclaim.Claim) { c.Holder.Stamp++ }, want: hostrelocate.ErrRefused}, |
| 679 | {name: "this box declared lost", stored: func(c *hostclaim.Claim) { c.Lost = append(c.Lost, "dst") }, want: hostrelocate.ErrRefused}, |
| 680 | {name: "the op stuck", stored: func(c *hostclaim.Claim) { c.Op.Journal["stuck"] = "an operator's" }, want: hostrelocate.ErrRefused}, |
| 681 | {name: "P opened", stored: func(c *hostclaim.Claim) { c.Op.Journal["prepared_opened"] = c.Op.Journal["prepared"] }, want: hostrelocate.ErrRefused}, |
| 682 | {name: "an observation at another install stamp", stale: func(_ *buildFixture, c *hostclaim.Claim) { |
| 683 | c.Op.Journal["install_stamp"] = strconv.FormatUint(c.Holder.Stamp+1, 10) |
| 684 | }, want: hostguard.ErrStampMismatch}, |
| 685 | {name: "an observation of another incarnation", stale: func(_ *buildFixture, c *hostclaim.Claim) { c.Holder.Incarnation = "another" }, want: hostguard.ErrStampMismatch}, |
| 686 | {name: "an observation naming another P", stale: func(_ *buildFixture, c *hostclaim.Claim) { c.Op.Journal["prepared"] = "p2" }, want: hostrelocate.ErrRefused}, |
| 687 | {name: "an observation naming another pin", stale: func(f *buildFixture, c *hostclaim.Claim) { |
| 688 | // The same pin bytes at another key: ReadPin accepts them, the |
| 689 | // journal's text differs. |
| 690 | var ref hostclaim.RecoveryPin |
| 691 | if err := json.Unmarshal([]byte(c.Op.Journal["pin"]), &ref); err != nil { |
| 692 | f.t.Fatal(err) |
| 693 | } |
| 694 | raw, _, err := f.claims.Get(context.Background(), ref.Key) |
| 695 | if err != nil { |
| 696 | f.t.Fatal(err) |
| 697 | } |
| 698 | ref.Key += ".copy" |
| 699 | if _, err := f.claims.Put(context.Background(), ref.Key, raw, ""); err != nil { |
| 700 | f.t.Fatal(err) |
| 701 | } |
| 702 | refRaw, _ := json.Marshal(ref) |
| 703 | c.Op.Journal["pin"] = string(refRaw) |
| 704 | }, want: hostrelocate.ErrRefused}, |
| 705 | } { |
| 706 | t.Run(tc.name, func(t *testing.T) { |
| 707 | f := newBuildFixture(t, true) |
| 708 | if tc.stored != nil { |
| 709 | o, err := hostclaim.Read(t.Context(), f.claims, buildHost) |
| 710 | if err != nil { |
| 711 | t.Fatal(err) |
| 712 | } |
| 713 | if f.obs, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error { tc.stored(c); return nil }); err != nil { |
| 714 | t.Fatal(err) |
| 715 | } |
| 716 | } |
| 717 | if tc.stale != nil { |
| 718 | f.obs = f.staleObs(func(c *hostclaim.Claim) { tc.stale(f, c) }) |
| 719 | } |
| 720 | before := f.bucket.Snapshot() |
| 721 | if err := f.build(); !errors.Is(err, tc.want) { |
| 722 | t.Fatalf("got %v, want %v", err, tc.want) |
| 723 | } |
| 724 | if f.bucket.Snapshot() != before { |
| 725 | t.Fatal("the bucket was written") |
| 726 | } |
| 727 | if tc.stored != nil { |
| 728 | if entries, _ := os.ReadDir(filepath.Join(f.a.Guard.DataDir, "hostguard")); len(entries) != 0 { |
| 729 | t.Fatalf("the guard was written: %v", entries) |
| 730 | } |
| 731 | } |
| 732 | }) |
| 733 | } |
| 734 | } |
| 735 | |
| 736 | // relocationBuildP checks its authority before it takes the replica lease |
| 737 | // (a stale build never writes the lease object on holder.location), again |
| 738 | // once the lease is held, before P's files manifest, and before the |
| 739 | // database build, so a refusal at each stops what follows it. |
| 740 | func TestRelocationBuildPChecksItsAuthorityBeforeEachStep(t *testing.T) { |
| 741 | refused := errors.New("sentinel: the authority refused") |
| 742 | for _, tc := range []struct { |
| 743 | name string |
| 744 | at int // the auth call that refuses |
| 745 | }{ |
| 746 | {"before the replica lease", 1}, |
| 747 | {"before P's files manifest", 2}, |
| 748 | {"before the database build", 3}, |
| 749 | } { |
| 750 | t.Run(tc.name, func(t *testing.T) { |
| 751 | f := newBuildFixture(t, true) |
| 752 | calls := 0 |
| 753 | auth := func(context.Context) error { |
| 754 | if calls++; calls == tc.at { |
| 755 | return refused |
| 756 | } |
| 757 | return nil |
| 758 | } |
| 759 | before := f.bucket.Snapshot() |
| 760 | if err := f.a.relocationBuildP(f.cfg)(t.Context(), f.obs, auth); !errors.Is(err, refused) || calls != tc.at { |
| 761 | t.Fatalf("got %v after %d checks", err, calls) |
| 762 | } |
| 763 | if tc.at == 1 && f.bucket.Snapshot() != before { |
| 764 | t.Fatal("the replica lease was taken without authority") |
| 765 | } |
| 766 | _, _, err := f.files.Get(t.Context(), hostfiles.ManifestKey(f.obs.Claim.Holder.Location+"/files/g-p1/")) |
| 767 | if wrote := err == nil; wrote != (tc.at > 2) { |
| 768 | t.Fatalf("P's files manifest written=%v (%v)", wrote, err) |
| 769 | } |
| 770 | if _, err := os.Stat(filepath.Join(f.cfg.WorkDir, "rel-1")); !errors.Is(err, os.ErrNotExist) { |
| 771 | t.Fatalf("the database build started: %v", err) |
| 772 | } |
| 773 | if f.installManifestExists() { |
| 774 | t.Fatal("the install manifest was written") |
| 775 | } |
| 776 | }) |
| 777 | } |
| 778 | } |
| 779 | |
| 780 | // lossyLease is a real replica lease whose loss the test signals. |
| 781 | type lossyLease struct { |
| 782 | *lease.Held |
| 783 | lost chan struct{} |
| 784 | } |
| 785 | |
| 786 | func (l lossyLease) Lost() <-chan struct{} { return l.lost } |
| 787 | |
| 788 | // The replica lease lost while the claim, the guard and main's Local all |
| 789 | // still admit the build stops it before the database build. |
| 790 | func TestRelocationBuildPStopsWhenTheReplicaLeaseIsLost(t *testing.T) { |
| 791 | f := newBuildFixture(t, true) |
| 792 | lost := make(chan struct{}) |
| 793 | f.a.acquireReplicaLease = func(ctx context.Context, bucket, prefix, region string) (replicaLease, error) { |
| 794 | h, err := lease.Acquire(ctx, bucket, prefix, region) |
| 795 | if err != nil { |
| 796 | return nil, err |
| 797 | } |
| 798 | return lossyLease{Held: h, lost: lost}, nil |
| 799 | } |
| 800 | calls := 0 |
| 801 | auth := func(ctx context.Context) error { |
| 802 | if calls++; calls == 3 { // after P's files manifest |
| 803 | close(lost) |
| 804 | <-ctx.Done() // the build has seen the loss |
| 805 | } |
| 806 | return nil |
| 807 | } |
| 808 | if err := f.a.relocationBuildP(f.cfg)(t.Context(), f.obs, auth); !errors.Is(err, context.Canceled) { |
| 809 | t.Fatalf("the build went on without its replica lease: %v", err) |
| 810 | } |
| 811 | if _, err := os.Stat(filepath.Join(f.cfg.WorkDir, "rel-1")); !errors.Is(err, os.ErrNotExist) { |
| 812 | t.Fatalf("the database build started: %v", err) |
| 813 | } |
| 814 | if f.installManifestExists() { |
| 815 | t.Fatal("the install manifest was written after the replica lease was lost") |
| 816 | } |
| 817 | } |
| 818 | |
| 819 | // Verify accepts only this pin's build of P: P's files manifest at |
| 820 | // sequence 0, and an install manifest of this op and host at the pin's N, |
| 821 | // whose integrity check passed and whose snapshot and anchor versions |
| 822 | // exist. Anything else built at P is refused, never reported done. |
| 823 | func TestPreparedRefusesAnythingButThisPinsBuild(t *testing.T) { |
| 824 | for _, tc := range []struct { |
| 825 | name string |
| 826 | files func(*hostfiles.Manifest) |
| 827 | im func(*buildFixture, *moves.InstallManifest) |
| 828 | }{ |
| 829 | {name: "P's files manifest not at sequence 0", files: func(m *hostfiles.Manifest) { m.Seq = 5 }}, |
| 830 | {name: "another op", im: func(_ *buildFixture, m *moves.InstallManifest) { m.Op = "rel-2" }}, |
| 831 | {name: "another host", im: func(_ *buildFixture, m *moves.InstallManifest) { m.Host = "other.example" }}, |
| 832 | {name: "another N", im: func(f *buildFixture, m *moves.InstallManifest) { |
| 833 | // A complete build at N-1: its snapshot and anchor exist, at |
| 834 | // their keys for N-1. Only the pin's N tells it apart. |
| 835 | copyAt := func(from, to string) string { |
| 836 | body, ok := f.bucket.Current("data", from) |
| 837 | if !ok { |
| 838 | f.t.Fatalf("%s is not there", from) |
| 839 | } |
| 840 | return f.bucket.Put("data", to, body) |
| 841 | } |
| 842 | n := m.N - 1 |
| 843 | m.SnapshotVersion = copyAt(m.SnapshotKey, moves.SnapshotKey(f.prefix(), n)) |
| 844 | m.AnchorVersion = copyAt(m.AnchorKey, moves.AnchorKey(f.prefix(), n)) |
| 845 | m.N, m.SnapshotKey, m.AnchorKey = n, moves.SnapshotKey(f.prefix(), n), moves.AnchorKey(f.prefix(), n) |
| 846 | }}, |
| 847 | {name: "a failed integrity check", im: func(_ *buildFixture, m *moves.InstallManifest) { m.IntegrityCheck = "corrupt" }}, |
| 848 | {name: "a snapshot version that is not there", im: func(_ *buildFixture, m *moves.InstallManifest) { m.SnapshotVersion = "missing" }}, |
| 849 | {name: "an anchor version that is not there", im: func(_ *buildFixture, m *moves.InstallManifest) { m.AnchorVersion = "missing" }}, |
| 850 | } { |
| 851 | t.Run(tc.name, func(t *testing.T) { |
| 852 | ctx := t.Context() |
| 853 | f := newBuildFixture(t, true) |
| 854 | if err := f.build(); err != nil { |
| 855 | t.Fatal(err) |
| 856 | } |
| 857 | if _, done, err := f.verifier.Verify(ctx, f.obs); !done || err != nil { |
| 858 | t.Fatalf("the untouched build: done=%v %v", done, err) |
| 859 | } |
| 860 | if tc.files != nil { |
| 861 | key := hostfiles.ManifestKey(f.obs.Claim.Holder.Location + "/files/g-p1/") |
| 862 | raw, etag, err := f.files.Get(ctx, key) |
| 863 | if err != nil { |
| 864 | t.Fatal(err) |
| 865 | } |
| 866 | var m hostfiles.Manifest |
| 867 | if err := json.Unmarshal(raw, &m); err != nil { |
| 868 | t.Fatal(err) |
| 869 | } |
| 870 | tc.files(&m) |
| 871 | if raw, err = json.Marshal(m); err != nil { |
| 872 | t.Fatal(err) |
| 873 | } |
| 874 | if _, err := f.files.Store.Put(ctx, key, raw, etag); err != nil { |
| 875 | t.Fatal(err) |
| 876 | } |
| 877 | } |
| 878 | if tc.im != nil { |
| 879 | im, _, err := readInstallManifest(ctx, f.bucket, "data", f.prefix()) |
| 880 | if err != nil { |
| 881 | t.Fatal(err) |
| 882 | } |
| 883 | tc.im(f, &im) |
| 884 | raw, err := json.Marshal(im) |
| 885 | if err != nil { |
| 886 | t.Fatal(err) |
| 887 | } |
| 888 | f.bucket.Put("data", moves.InstallManifestKey(f.prefix()), raw) |
| 889 | } |
| 890 | if _, done, err := f.verifier.Verify(ctx, f.obs); done || !errors.Is(err, hostrelocate.ErrRefused) { |
| 891 | t.Fatalf("done=%v err=%v", done, err) |
| 892 | } |
| 893 | }) |
| 894 | } |
| 895 | } |
| 896 | |
amadan