| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "crypto/rand" |
| 7 | "errors" |
| 8 | "path/filepath" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/hostactors" |
| 14 | "github.com/carlosframework/platform/internal/provision" |
| 15 | "github.com/carlosframework/platform/internal/registry" |
| 16 | "github.com/carlosframework/platform/internal/release" |
| 17 | "github.com/carlosframework/platform/internal/seeds" |
| 18 | ) |
| 19 | |
| 20 | // activation is a deployment at the moment claim activation publishes its |
| 21 | // backfill: a signed roster of two boxes, the collecting manifest (the |
| 22 | // record-writer freeze), and one signed census report per box. |
| 23 | type activation struct { |
| 24 | t *testing.T |
| 25 | ctx context.Context |
| 26 | f *fixture |
| 27 | sg release.Signer |
| 28 | d hostactors.Directory |
| 29 | roster hostactors.Roster |
| 30 | boxKeys map[string]ed25519.PrivateKey |
| 31 | records []Record |
| 32 | } |
| 33 | |
| 34 | func newActivation(t *testing.T) *activation { |
| 35 | t.Helper() |
| 36 | a := &activation{t: t, ctx: context.Background(), f: newFixture(t), boxKeys: map[string]ed25519.PrivateKey{}} |
| 37 | pub, key, err := ed25519.GenerateKey(rand.Reader) |
| 38 | if err != nil { |
| 39 | t.Fatal(err) |
| 40 | } |
| 41 | a.sg = release.RootSigner(key) |
| 42 | a.roster = hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Regions: map[string]hostactors.Region{"ie": {}}, |
| 43 | Boxes: map[string]hostactors.Box{}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} |
| 44 | for i, ref := range []string{"box-a", "box-b"} { |
| 45 | boxPub, boxKey, err := ed25519.GenerateKey(rand.Reader) |
| 46 | if err != nil { |
| 47 | t.Fatal(err) |
| 48 | } |
| 49 | a.boxKeys[ref] = boxKey |
| 50 | a.roster.Boxes[ref] = hostactors.Box{CensusPublicKey: boxPub, Enrollment: strings.Repeat(string(rune('a'+i)), 32), Namespace: "platform", BoxID: ref, Label: ref, |
| 51 | Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "ie", InstanceID: "i-" + ref}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"} |
| 52 | } |
| 53 | raw, err := release.SignRootEnvelope(a.sg, a.roster) |
| 54 | if err != nil { |
| 55 | t.Fatal(err) |
| 56 | } |
| 57 | if _, err := a.f.store.Put(a.ctx, hostactors.Key, raw, ""); err != nil { |
| 58 | t.Fatal(err) |
| 59 | } |
| 60 | a.d = hostactors.Directory{Store: a.f.store, PublicKey: pub, Bucket: "bucket"} |
| 61 | return a |
| 62 | } |
| 63 | |
| 64 | // record writes one instance record, with seed as its seed (nil for none). |
| 65 | func (a *activation) record(host string, seed *provision.SeedSource) { |
| 66 | a.t.Helper() |
| 67 | r, err := PrepareInstance(provision.Record{Schema: 1, Account: "bab", App: "shop", Host: host, Slug: strings.SplitN(host, ".", 2)[0], |
| 68 | Channel: "edge", Region: "ie", Acquisition: "acquisition", Seed: seed}, a.f.now, "ie") |
| 69 | if err != nil { |
| 70 | a.t.Fatal(err) |
| 71 | } |
| 72 | if _, err := a.f.store.Put(a.ctx, r.Holder.Key, r.Body, ""); err != nil { |
| 73 | a.t.Fatal(err) |
| 74 | } |
| 75 | if r, err = ReadRecord(a.ctx, a.f.store, r.Holder.Key); err != nil { |
| 76 | a.t.Fatalf("the fixture's record does not read: %v", err) |
| 77 | } |
| 78 | a.records = append(a.records, r) |
| 79 | } |
| 80 | |
| 81 | // publish writes the collecting manifest over the records, each box's |
| 82 | // signed report carrying its seed evidence (a missing entry is a report |
| 83 | // with none), and runs activation's Publish under the manifest's etag. |
| 84 | func (a *activation) publish(evidence map[string]*CensusSeeds) error { |
| 85 | a.t.Helper() |
| 86 | p := BackfillPlan{V: 1, ID: "backfill", Phase: "collecting", DirectoryRevision: 1, CreatedAt: a.f.now, Hosts: map[string]BackfillHost{}} |
| 87 | for _, r := range a.records { |
| 88 | p.Hosts[r.Holder.Host] = BackfillHost{Key: r.Holder.Key, ETag: r.ETag, Acquisition: "acquisition", Region: "ie", Box: "box-a"} |
| 89 | } |
| 90 | raw, err := release.SignRootEnvelope(a.sg, p) |
| 91 | if err != nil { |
| 92 | a.t.Fatal(err) |
| 93 | } |
| 94 | tag, err := a.f.store.Put(a.ctx, BackfillKey, raw, "") |
| 95 | if err != nil { |
| 96 | a.t.Fatal(err) |
| 97 | } |
| 98 | for ref, box := range a.roster.Boxes { |
| 99 | report := CensusReport{Box: ref, Enrollment: box.Enrollment, BootID: "boot", Reservation: p.ID, DirectoryRevision: 1, |
| 100 | CapturedAt: a.f.now, Complete: true, Installations: []CensusInstallation{}, Seeds: evidence[ref]} |
| 101 | if err := PublishCensusReport(a.ctx, a.f.store, a.d, report, a.boxKeys[ref]); err != nil { |
| 102 | a.t.Fatalf("publish %s's census report: %v", ref, err) |
| 103 | } |
| 104 | } |
| 105 | b := &Backfill{Store: a.f.store, Directory: a.d} |
| 106 | return b.Publish(a.ctx, a.sg, tag) |
| 107 | } |
| 108 | |
| 109 | // phase is the backfill manifest's phase now. |
| 110 | func (a *activation) phase() string { |
| 111 | a.t.Helper() |
| 112 | b := &Backfill{Store: a.f.store, Directory: a.d} |
| 113 | p, _, err := b.readPlan(a.ctx) |
| 114 | if err != nil { |
| 115 | a.t.Fatal(err) |
| 116 | } |
| 117 | return p.Phase |
| 118 | } |
| 119 | |
| 120 | func noSeeds() map[string]*CensusSeeds { |
| 121 | return map[string]*CensusSeeds{"box-a": {Journaled: []string{}, Gated: []string{}}, "box-b": {Journaled: []string{}, Gated: []string{}}} |
| 122 | } |
| 123 | |
| 124 | func testSeed(kind, state string) *provision.SeedSource { |
| 125 | id, _ := seeds.NewID() |
| 126 | s := &provision.SeedSource{ID: id, Kind: kind, State: state, SHA256: strings.Repeat("a", 64), Bytes: 10, By: "t", At: time.Date(2026, 10, 5, 11, 0, 0, 0, time.UTC)} |
| 127 | if kind == seeds.KindImport { |
| 128 | s.Replaces, s.VerifyHash = "127.0.0.1:8401", strings.Repeat("b", 64) |
| 129 | } else { |
| 130 | s.From = "src.shop.example.com" |
| 131 | } |
| 132 | return s |
| 133 | } |
| 134 | |
| 135 | // refused asserts err is activation's seed refusal over base, naming what. |
| 136 | func refused(t *testing.T, err, base error, what string) { |
| 137 | t.Helper() |
| 138 | var se *SeedActivationError |
| 139 | if !errors.As(err, &se) || !errors.Is(err, base) || !strings.Contains(err.Error(), what) { |
| 140 | t.Fatalf("Publish = %v; want a seed refusal over %v naming %q", err, base, what) |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | // Merge review S2 (a): with every box reporting no seed state and no |
| 145 | // record carrying an unfinished seed, activation publishes as before. A |
| 146 | // finished seed does not hold it. |
| 147 | func TestActivationPublishesWithNoUnfinishedSeed(t *testing.T) { |
| 148 | a := newActivation(t) |
| 149 | a.record("plain.shop.example.com", nil) |
| 150 | a.record("copy.shop.example.com", testSeed(seeds.KindReplica, seeds.StatePlaced)) |
| 151 | a.record("moved.shop.example.com", testSeed(seeds.KindImport, seeds.StateCommitted)) |
| 152 | if err := a.publish(noSeeds()); err != nil { |
| 153 | t.Fatalf("Publish: %v", err) |
| 154 | } |
| 155 | if got := a.phase(); got != "adopting" { |
| 156 | t.Fatalf("phase = %q, want adopting", got) |
| 157 | } |
| 158 | } |
| 159 | |
| 160 | // Activation refuses over each unfinished seed a record can carry, under |
| 161 | // the collecting freeze, and publishes nothing. That includes the two |
| 162 | // states ruled in after the first fix: a --from seed at ready, which |
| 163 | // nothing can place once claims are on, and an import at refused, which a |
| 164 | // claim console cannot delete. Only a placed copy and a committed import |
| 165 | // publish. |
| 166 | func TestActivationRefusesAnUnfinishedSeedRecord(t *testing.T) { |
| 167 | for _, c := range []struct { |
| 168 | kind, state string |
| 169 | refuse bool |
| 170 | }{ |
| 171 | {seeds.KindReplica, seeds.StateReserving, true}, |
| 172 | {seeds.KindImport, seeds.StateAwaitingUpload, true}, |
| 173 | {seeds.KindImport, seeds.StateUploaded, true}, |
| 174 | {seeds.KindImport, seeds.StateReady, true}, |
| 175 | {seeds.KindImport, seeds.StateSwapped, true}, |
| 176 | {seeds.KindImport, seeds.StatePlaced, true}, |
| 177 | {seeds.KindImport, seeds.StateAborted, true}, |
| 178 | {seeds.KindReplica, seeds.StateAborted, true}, |
| 179 | {seeds.KindImport, seeds.StateRolledBack, true}, |
| 180 | {seeds.KindReplica, seeds.StateRolledBack, true}, |
| 181 | {seeds.KindReplica, seeds.StateReady, true}, |
| 182 | {seeds.KindImport, seeds.StateRefused, true}, |
| 183 | {seeds.KindReplica, seeds.StateCommitted, true}, |
| 184 | {"later", seeds.StatePlaced, true}, |
| 185 | {"later", seeds.StateCommitted, true}, |
| 186 | {seeds.KindReplica, seeds.StatePlaced, false}, |
| 187 | {seeds.KindImport, seeds.StateCommitted, false}, |
| 188 | } { |
| 189 | t.Run(c.kind+" "+c.state, func(t *testing.T) { |
| 190 | a := newActivation(t) |
| 191 | a.record("plain.shop.example.com", nil) |
| 192 | a.record("seeded.shop.example.com", testSeed(c.kind, c.state)) |
| 193 | err := a.publish(noSeeds()) |
| 194 | if !c.refuse { |
| 195 | if err != nil || a.phase() != "adopting" { |
| 196 | t.Fatalf("Publish = %v, phase %q; want published", err, a.phase()) |
| 197 | } |
| 198 | return |
| 199 | } |
| 200 | refused(t, err, ErrPending, "seeded.shop.example.com has an unfinished seed ("+c.kind+", "+c.state+")") |
| 201 | if got := a.phase(); got != "collecting" { |
| 202 | t.Fatalf("phase = %q after a refusal, want collecting", got) |
| 203 | } |
| 204 | }) |
| 205 | } |
| 206 | } |
| 207 | |
| 208 | // Activation refuses when a box's signed report carries no seed state: it |
| 209 | // fails closed, whatever the other boxes and the records say. |
| 210 | func TestActivationRefusesWithoutABoxesSeedEvidence(t *testing.T) { |
| 211 | for name, evidence := range map[string]map[string]*CensusSeeds{ |
| 212 | "no box says": {}, |
| 213 | "one box says": {"box-a": {Journaled: []string{}, Gated: []string{}}}, |
| 214 | "the other one": {"box-b": {Journaled: []string{}, Gated: []string{}}}, |
| 215 | } { |
| 216 | t.Run(name, func(t *testing.T) { |
| 217 | a := newActivation(t) |
| 218 | a.record("plain.shop.example.com", nil) |
| 219 | err := a.publish(evidence) |
| 220 | refused(t, err, ErrIdentity, "reported no seed state") |
| 221 | if got := a.phase(); got != "collecting" { |
| 222 | t.Fatalf("phase = %q after a refusal, want collecting", got) |
| 223 | } |
| 224 | }) |
| 225 | } |
| 226 | // A collector that cannot read its registry leaves the evidence out. |
| 227 | if seeds, err := CollectCensusSeeds(failingSeedState{}); err == nil || seeds != nil { |
| 228 | t.Fatalf("CollectCensusSeeds over a registry that cannot be read = %+v, %v", seeds, err) |
| 229 | } |
| 230 | } |
| 231 | |
| 232 | type failingSeedState struct{} |
| 233 | |
| 234 | func (failingSeedState) List() ([]registry.Route, error) { return nil, errors.New("registry busy") } |
| 235 | func (failingSeedState) ImportJournalHosts() ([]string, error) { |
| 236 | return nil, errors.New("registry busy") |
| 237 | } |
| 238 | |
| 239 | // importOnBox is a box's registry holding one import, taken as far as |
| 240 | // state: swapped (the gated row and a journal row), rolling-back (the gate |
| 241 | // closed), or cleanup-pending (the original route back, the journal row |
| 242 | // still there). |
| 243 | func importOnBox(t *testing.T, host, state string) *registry.Registry { |
| 244 | t.Helper() |
| 245 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 246 | if err != nil { |
| 247 | t.Fatal(err) |
| 248 | } |
| 249 | t.Cleanup(func() { reg.Close() }) |
| 250 | if err := reg.Upsert(registry.Route{Host: host, Kind: "service", Addr: "127.0.0.1:8401", Status: "ready", App: "shop", Account: "bab", CreatedAt: 1700000000}); err != nil { |
| 251 | t.Fatal(err) |
| 252 | } |
| 253 | original, err := reg.Get(host) |
| 254 | if err != nil || original == nil { |
| 255 | t.Fatal(original, err) |
| 256 | } |
| 257 | id, _ := seeds.NewID() |
| 258 | next := registry.Route{Host: host, Kind: "instance", Socket: "/run/carlos/instances/x/app.sock", DBPath: "/data/instances/x/" + host + ".db", |
| 259 | Status: "ready", App: "shop", Account: "bab", Hibernate: true, Provisioned: true, ImportGate: strings.Repeat("b", 64), |
| 260 | SeedSource: seeds.RowSeed{ID: id, Kind: seeds.KindImport, State: seeds.StateSwapped}.Encode(), CreatedAt: 1700000001} |
| 261 | if err := reg.SwapForImport(registry.ImportJournal{Host: host, SeedID: id, Original: *original, RecordKey: provision.RecordKey("bab", "shop", host), At: 1, |
| 262 | EnvBefore: registry.EnvBeforeAbsent}, next); err != nil { |
| 263 | t.Fatal(err) |
| 264 | } |
| 265 | if state == registry.JournalSwapped { |
| 266 | return reg |
| 267 | } |
| 268 | if err := reg.CloseImportGate(host, id, 7); err != nil { |
| 269 | t.Fatal(err) |
| 270 | } |
| 271 | if state == registry.JournalRollingBack { |
| 272 | return reg |
| 273 | } |
| 274 | if err := reg.RestoreImportRoute(host, id); err != nil { |
| 275 | t.Fatal(err) |
| 276 | } |
| 277 | return reg |
| 278 | } |
| 279 | |
| 280 | // The ruling's four states, with the box's evidence read from a real |
| 281 | // registry by the collector's own function. swapped and placed are an |
| 282 | // import the record still shows; rolling-back and cleanup-pending are |
| 283 | // shown here with the record already gone, where only the box's signed |
| 284 | // evidence can say an import is still in flight. |
| 285 | func TestActivationRefusesAnImportInFlightOnABox(t *testing.T) { |
| 286 | const host = "old.shop.example.com" |
| 287 | for _, c := range []struct { |
| 288 | name string |
| 289 | record string // the record's seed state, "" for no record |
| 290 | journal string |
| 291 | wantJournaled bool |
| 292 | wantGated bool |
| 293 | }{ |
| 294 | {"swapped", seeds.StateSwapped, registry.JournalSwapped, true, true}, |
| 295 | {"placed", seeds.StatePlaced, registry.JournalSwapped, true, true}, |
| 296 | {"rolling-back", "", registry.JournalRollingBack, true, true}, |
| 297 | {"cleanup-pending", "", registry.JournalCleanupPending, true, false}, |
| 298 | } { |
| 299 | t.Run(c.name, func(t *testing.T) { |
| 300 | a := newActivation(t) |
| 301 | a.record("plain.shop.example.com", nil) |
| 302 | if c.record != "" { |
| 303 | a.record(host, testSeed(seeds.KindImport, c.record)) |
| 304 | } |
| 305 | evidence := noSeeds() |
| 306 | got, err := CollectCensusSeeds(importOnBox(t, host, c.journal)) |
| 307 | if err != nil { |
| 308 | t.Fatal(err) |
| 309 | } |
| 310 | if (len(got.Journaled) == 1 && got.Journaled[0] == host) != c.wantJournaled || (len(got.Gated) == 1 && got.Gated[0] == host) != c.wantGated { |
| 311 | t.Fatalf("the box's evidence = %+v; want journaled %v, gated %v", got, c.wantJournaled, c.wantGated) |
| 312 | } |
| 313 | evidence["box-b"] = got |
| 314 | refused(t, a.publish(evidence), ErrPending, "box box-b still holds an import journal row for "+host) |
| 315 | if got := a.phase(); got != "collecting" { |
| 316 | t.Fatalf("phase = %q after a refusal, want collecting", got) |
| 317 | } |
| 318 | }) |
| 319 | } |
| 320 | // A gate with no journal row (a committed import whose box has not yet |
| 321 | // cleared its gate cannot be told from one that never will) refuses too. |
| 322 | a := newActivation(t) |
| 323 | a.record("plain.shop.example.com", nil) |
| 324 | evidence := noSeeds() |
| 325 | evidence["box-a"] = &CensusSeeds{Journaled: []string{}, Gated: []string{host}} |
| 326 | refused(t, a.publish(evidence), ErrPending, "box box-a still gates "+host+" for an import") |
| 327 | } |
| 328 | |
| 329 | // A box with no import reports two empty lists, which is not "did not |
| 330 | // say": the collector's answer for a clean registry lets activation go on. |
| 331 | func TestCollectCensusSeedsOnACleanBox(t *testing.T) { |
| 332 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 333 | if err != nil { |
| 334 | t.Fatal(err) |
| 335 | } |
| 336 | defer reg.Close() |
| 337 | if err := reg.Upsert(registry.Route{Host: "plain.shop.example.com", Kind: "instance", Socket: "/run/x.sock", App: "shop", Account: "bab", Hibernate: true}); err != nil { |
| 338 | t.Fatal(err) |
| 339 | } |
| 340 | got, err := CollectCensusSeeds(reg) |
| 341 | if err != nil || got == nil || got.Journaled == nil || got.Gated == nil || len(got.Journaled)+len(got.Gated) != 0 { |
| 342 | t.Fatalf("CollectCensusSeeds = %+v, %v; want two empty lists", got, err) |
| 343 | } |
| 344 | a := newActivation(t) |
| 345 | a.record("plain.shop.example.com", nil) |
| 346 | if err := a.publish(map[string]*CensusSeeds{"box-a": got, "box-b": got}); err != nil { |
| 347 | t.Fatalf("Publish with the collector's clean evidence: %v", err) |
| 348 | } |
| 349 | } |
| 350 | |
amadan