| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "crypto/rand" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 9 | "encoding/json" |
| 10 | "errors" |
| 11 | "reflect" |
| 12 | "strings" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/domains" |
| 17 | "github.com/carlosframework/platform/internal/hostactors" |
| 18 | "github.com/carlosframework/platform/internal/hostclaim" |
| 19 | "github.com/carlosframework/platform/internal/release" |
| 20 | ) |
| 21 | |
| 22 | func TestReconcileFrozenBackfillAndQuarantine(t *testing.T) { |
| 23 | testReconcileFrozenBackfill(t, false, "") |
| 24 | } |
| 25 | |
| 26 | func TestReconcileFrozenBackfillRepairsLegacyHolder(t *testing.T) { |
| 27 | testReconcileFrozenBackfill(t, true, "") |
| 28 | } |
| 29 | |
| 30 | func testReconcileFrozenBackfill(t *testing.T, legacy bool, scenario string) { |
| 31 | t.Helper() |
| 32 | ctx := context.Background() |
| 33 | f := newFixture(t) |
| 34 | pub, key, err := ed25519.GenerateKey(rand.Reader) |
| 35 | if err != nil { |
| 36 | t.Fatal(err) |
| 37 | } |
| 38 | sg := release.RootSigner(key) |
| 39 | roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Regions: map[string]hostactors.Region{"ie": {}, "us": {}}, Boxes: map[string]hostactors.Box{}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} |
| 40 | boxKeys := map[string]ed25519.PrivateKey{} |
| 41 | for i, ref := range []string{"box-a", "box-b"} { |
| 42 | boxPub, boxKey, e := ed25519.GenerateKey(rand.Reader) |
| 43 | if e != nil { |
| 44 | t.Fatal(e) |
| 45 | } |
| 46 | boxKeys[ref] = boxKey |
| 47 | region := "ie" |
| 48 | en := strings.Repeat("a", 32) |
| 49 | if i == 1 { |
| 50 | region = "us" |
| 51 | en = strings.Repeat("b", 32) |
| 52 | } |
| 53 | roster.Boxes[ref] = hostactors.Box{CensusPublicKey: boxPub, Enrollment: en, Namespace: "platform", BoxID: ref, Label: ref, Region: region, Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: region, InstanceID: "i-" + ref}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"} |
| 54 | } |
| 55 | if scenario == "two-box-region" || scenario == "two-owners" { |
| 56 | box := roster.Boxes["box-b"] |
| 57 | box.Region, box.Cloud.Region = "ie", "ie" |
| 58 | roster.Boxes["box-b"] = box |
| 59 | } |
| 60 | if scenario == "inactive-owner" || scenario == "wrong-region" { |
| 61 | box := roster.Boxes["box-a"] |
| 62 | if scenario == "inactive-owner" { |
| 63 | box.State = "retired" |
| 64 | } else { |
| 65 | box.Region, box.Cloud.Region = "us", "us" |
| 66 | } |
| 67 | roster.Boxes["box-a"] = box |
| 68 | } |
| 69 | put := func(k string, v any, signed bool) string { |
| 70 | t.Helper() |
| 71 | var raw []byte |
| 72 | var e error |
| 73 | if signed { |
| 74 | raw, e = release.SignRootEnvelope(sg, v) |
| 75 | } else { |
| 76 | raw, e = json.Marshal(v) |
| 77 | } |
| 78 | if e != nil { |
| 79 | t.Fatal(e) |
| 80 | } |
| 81 | tag, e := f.store.Put(ctx, k, raw, "") |
| 82 | if e != nil { |
| 83 | t.Fatal(e) |
| 84 | } |
| 85 | return tag |
| 86 | } |
| 87 | put(hostactors.Key, roster, true) |
| 88 | d := hostactors.Directory{Store: f.store, PublicKey: pub, Bucket: "bucket"} |
| 89 | c := NewBackfillController(f.store, d, func() time.Time { return f.now }, f.c.NewID) |
| 90 | r := f.record(t, "legacy.example.com") |
| 91 | var body map[string]any |
| 92 | if err = json.Unmarshal(r.Body, &body); err != nil { |
| 93 | t.Fatal(err) |
| 94 | } |
| 95 | body["acq"] = "ownership-acquisition" |
| 96 | put(r.Holder.Key, body, false) |
| 97 | r, err = ReadRecord(ctx, f.store, r.Holder.Key) |
| 98 | if err != nil { |
| 99 | t.Fatal(err) |
| 100 | } |
| 101 | inc, err := r.Incarnation() |
| 102 | if err != nil { |
| 103 | t.Fatal(err) |
| 104 | } |
| 105 | put(domains.ClaimKey(r.Holder.Host), domains.Claim{Schema: 1, Host: r.Holder.Host, Account: "bab", App: "jam", Acquisition: "ownership-acquisition", RecordIncarnation: inc, OwnershipState: "committed"}, false) |
| 106 | if err = c.ReconcileClaims(ctx); err != nil { |
| 107 | t.Fatalf("unfrozen adoption: %v", err) |
| 108 | } |
| 109 | obs, err := hostclaim.Read(ctx, f.store, r.Holder.Host) |
| 110 | if err != nil || !obs.Absent { |
| 111 | t.Fatal(obs, err) |
| 112 | } |
| 113 | var prior hostclaim.Observed |
| 114 | if legacy { |
| 115 | nh := r.Holder |
| 116 | nh.Acquisition = "" |
| 117 | nh.Region = "ie" |
| 118 | prior, err = hostclaim.Update(ctx, f.store, obs, func(claim *hostclaim.Claim) error { |
| 119 | if err := hostclaim.Adopt(nh, "box-a", []string{"us"}, []string{"box-b"}, "legacy-discovery", f.c.NewID)(claim); err != nil { |
| 120 | return err |
| 121 | } |
| 122 | claim.Holder.Stamp = 17 |
| 123 | claim.Holder.Generation = "legacy-generation" |
| 124 | return nil |
| 125 | }) |
| 126 | if err != nil { |
| 127 | t.Fatal(err) |
| 128 | } |
| 129 | } |
| 130 | p := BackfillPlan{V: 1, ID: "backfill", Phase: "adopting", DirectoryRevision: 1, CreatedAt: f.now, Reports: map[string]string{}, Hosts: map[string]BackfillHost{r.Holder.Host: {Key: r.Holder.Key, ETag: r.ETag, Acquisition: "ownership-acquisition", Region: "ie", Box: "box-a"}}} |
| 131 | for ref, b := range roster.Boxes { |
| 132 | report := CensusReport{Box: ref, Enrollment: b.Enrollment, BootID: "boot", Reservation: p.ID, DirectoryRevision: 1, CapturedAt: f.now, Complete: true, Installations: []CensusInstallation{}} |
| 133 | if ref == "box-a" || scenario == "two-owners" { |
| 134 | report.Installations = append(report.Installations, CensusInstallation{Host: r.Holder.Host, Account: "bab", App: "jam", Family: "instance", Key: r.Holder.Key, Incarnation: inc, Local: true, Route: true}) |
| 135 | } |
| 136 | key, _ := reportKey(p.ID, ref) |
| 137 | body, e := json.Marshal(report) |
| 138 | if e != nil { |
| 139 | t.Fatal(e) |
| 140 | } |
| 141 | put(key, CensusEnvelope{Report: report, Signature: ed25519.Sign(boxKeys[ref], body)}, false) |
| 142 | raw, _, err := f.store.Get(ctx, key) |
| 143 | if err != nil { |
| 144 | t.Fatal(err) |
| 145 | } |
| 146 | sum := sha256.Sum256(raw) |
| 147 | p.Reports[ref] = hex.EncodeToString(sum[:]) |
| 148 | } |
| 149 | put(BackfillKey, p, true) |
| 150 | qtag := put(HostsQuarantineKey, HostsQuarantine{V: 1, Hosts: map[string]string{r.Holder.Host: "conflict"}}, false) |
| 151 | if err = c.ReconcileClaims(ctx); !errors.Is(err, ErrPending) { |
| 152 | t.Fatalf("quarantine: %v", err) |
| 153 | } |
| 154 | raw, _ := json.Marshal(HostsQuarantine{V: 1, Hosts: map[string]string{}}) |
| 155 | if _, err = f.store.Put(ctx, HostsQuarantineKey, raw, qtag); err != nil { |
| 156 | t.Fatal(err) |
| 157 | } |
| 158 | |
| 159 | // A digest listed by a root manifest cannot turn a forged box report into |
| 160 | // an authenticated census. Test the verifier before its cache is populated. |
| 161 | reportPath, _ := reportKey(p.ID, "box-a") |
| 162 | authentic, reportTag, e := f.store.Get(ctx, reportPath) |
| 163 | if e != nil { |
| 164 | t.Fatal(e) |
| 165 | } |
| 166 | var forged CensusEnvelope |
| 167 | if e = json.Unmarshal(authentic, &forged); e != nil { |
| 168 | t.Fatal(e) |
| 169 | } |
| 170 | forged.Signature[0] ^= 1 |
| 171 | forgedRaw, _ := json.Marshal(forged) |
| 172 | forgedTag, e := f.store.Put(ctx, reportPath, forgedRaw, reportTag) |
| 173 | if e != nil { |
| 174 | t.Fatal(e) |
| 175 | } |
| 176 | sum := sha256.Sum256(forgedRaw) |
| 177 | p.Reports["box-a"] = hex.EncodeToString(sum[:]) |
| 178 | _, manifestTag, e := f.store.Get(ctx, BackfillKey) |
| 179 | if e != nil { |
| 180 | t.Fatal(e) |
| 181 | } |
| 182 | signed, _ := release.SignRootEnvelope(sg, p) |
| 183 | manifestTag, e = f.store.Put(ctx, BackfillKey, signed, manifestTag) |
| 184 | if e != nil { |
| 185 | t.Fatal(e) |
| 186 | } |
| 187 | if e = c.ReconcileClaims(ctx); e == nil { |
| 188 | t.Fatal("accepted forged census") |
| 189 | } |
| 190 | if _, e = f.store.Put(ctx, reportPath, authentic, forgedTag); e != nil { |
| 191 | t.Fatal(e) |
| 192 | } |
| 193 | sum = sha256.Sum256(authentic) |
| 194 | p.Reports["box-a"] = hex.EncodeToString(sum[:]) |
| 195 | signed, _ = release.SignRootEnvelope(sg, p) |
| 196 | if _, e = f.store.Put(ctx, BackfillKey, signed, manifestTag); e != nil { |
| 197 | t.Fatal(e) |
| 198 | } |
| 199 | if scenario == "two-owners" || scenario == "inactive-owner" || scenario == "wrong-region" { |
| 200 | if err = c.ReconcileClaims(ctx); !errors.Is(err, ErrIdentity) { |
| 201 | t.Fatalf("ineligible census owner: %v", err) |
| 202 | } |
| 203 | obs, err = hostclaim.Read(ctx, f.store, r.Holder.Host) |
| 204 | if err != nil || !obs.Absent { |
| 205 | t.Fatalf("ineligible owner adopted: %+v %v", obs, err) |
| 206 | } |
| 207 | return |
| 208 | } |
| 209 | if err = c.ReconcileClaims(ctx); err != nil { |
| 210 | t.Fatal(err) |
| 211 | } |
| 212 | obs, err = hostclaim.Read(ctx, f.store, r.Holder.Host) |
| 213 | if err != nil { |
| 214 | t.Fatal(err) |
| 215 | } |
| 216 | if obs.Absent || obs.Claim.Holder.Acquisition != "ownership-acquisition" || obs.Claim.Op.Exec != "box-a" || obs.Claim.Touched.Complete || len(obs.Claim.Obligations) != 2 { |
| 217 | t.Fatalf("bad backfill: %+v", obs.Claim) |
| 218 | } |
| 219 | wantDestinations := map[string]hostclaim.Obligation{ |
| 220 | "park-replica": {Region: "us", Owner: "region:us"}, |
| 221 | "park-local": {Box: "box-b", Owner: "box:box-b"}, |
| 222 | } |
| 223 | for _, obligation := range obs.Claim.Obligations { |
| 224 | want, ok := wantDestinations[obligation.Kind] |
| 225 | if !ok || obligation.Region != want.Region || obligation.Box != want.Box || obligation.Owner != want.Owner { |
| 226 | t.Fatalf("wrong discovery destination: %+v; want %+v", obligation, want) |
| 227 | } |
| 228 | delete(wantDestinations, obligation.Kind) |
| 229 | } |
| 230 | if len(wantDestinations) != 0 { |
| 231 | t.Fatalf("missing discovery destinations: %+v", wantDestinations) |
| 232 | } |
| 233 | if legacy { |
| 234 | want := *prior.Claim |
| 235 | holder := *want.Holder |
| 236 | holder.Acquisition = "ownership-acquisition" |
| 237 | want.Holder = &holder |
| 238 | want.Gen++ |
| 239 | if obs.ETag == prior.ETag || !reflect.DeepEqual(obs.Claim, &want) { |
| 240 | t.Fatalf("legacy repair changed more than acquisition and claim generation: got %+v (holder %+v), want %+v (holder %+v)", obs.Claim, obs.Claim.Holder, &want, want.Holder) |
| 241 | } |
| 242 | } |
| 243 | before := obs.ETag |
| 244 | if err = c.ReconcileClaims(ctx); err != nil { |
| 245 | t.Fatal(err) |
| 246 | } |
| 247 | obs, err = hostclaim.Read(ctx, f.store, r.Holder.Host) |
| 248 | if err != nil || obs.ETag != before { |
| 249 | t.Fatal("non-idempotent", err) |
| 250 | } |
| 251 | } |
| 252 | func TestReconcileDiscardPartialCensus(t *testing.T) { |
| 253 | f := newFixture(t) |
| 254 | r := f.record(t, "legacy.example.com") |
| 255 | if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil { |
| 256 | t.Fatal(err) |
| 257 | } |
| 258 | fail := errors.New("second listing failed") |
| 259 | f.store.afterList = func(prefix string, keys []string) error { |
| 260 | if strings.Contains(prefix, "sidecars/") { |
| 261 | return fail |
| 262 | } |
| 263 | return nil |
| 264 | } |
| 265 | calls := 0 |
| 266 | err := ReconcileClaims(f.ctx, f.c, func(context.Context, string) (hostclaim.Observed, error) { calls++; return hostclaim.Observed{}, nil }) |
| 267 | if !errors.Is(err, fail) || calls != 0 { |
| 268 | t.Fatal(err, calls) |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | func TestClaimDeleteDoorNeverAdopts(t *testing.T) { |
| 273 | f := newFixture(t) |
| 274 | r := f.record(t, "legacy.example.com") |
| 275 | if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil { |
| 276 | t.Fatal(err) |
| 277 | } |
| 278 | inc, err := r.Incarnation() |
| 279 | if err != nil { |
| 280 | t.Fatal(err) |
| 281 | } |
| 282 | if _, err = f.c.BeginRetire(f.ctx, r.Holder.Key, "test", inc); !errors.Is(err, ErrPending) { |
| 283 | t.Fatal(err) |
| 284 | } |
| 285 | obs, err := hostclaim.Read(f.ctx, f.store, r.Holder.Host) |
| 286 | if err != nil || !obs.Absent { |
| 287 | t.Fatalf("door adopted: %+v %v", obs, err) |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | func TestBackfillAuthorityCannotGainMutationMethods(t *testing.T) { |
| 292 | c := NewBackfillController(nil, hostactors.Directory{}, time.Now, func() string { return "id" }) |
| 293 | if _, ok := c.Authority.(interface { |
| 294 | StartCreate(context.Context, hostclaim.Observed, Record, string, string) (hostclaim.Observed, error) |
| 295 | }); ok { |
| 296 | t.Fatal("backfill gained create authority") |
| 297 | } |
| 298 | if _, ok := c.Authority.(interface { |
| 299 | StartDeleteClaim(context.Context, hostclaim.Observed, Record, string) (hostclaim.Observed, error) |
| 300 | }); ok { |
| 301 | t.Fatal("backfill gained delete authority") |
| 302 | } |
| 303 | if _, ok := c.Authority.(interface { |
| 304 | RecordEffects(context.Context, hostclaim.Observed, Record) error |
| 305 | }); ok { |
| 306 | t.Fatal("backfill gained effect authority") |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | func TestReconcileBackfillCensusOwnerEligibility(t *testing.T) { |
| 311 | for _, scenario := range []string{"two-box-region", "two-owners", "inactive-owner", "wrong-region"} { |
| 312 | t.Run(scenario, func(t *testing.T) { |
| 313 | testReconcileFrozenBackfill(t, false, scenario) |
| 314 | }) |
| 315 | } |
| 316 | } |
| 317 | |
| 318 | func TestReconcileBackfillPlanPhases(t *testing.T) { |
| 319 | for _, phase := range []string{"absent", "collecting", "done", "future", "adopting", "bad-signature", "read-error"} { |
| 320 | t.Run(phase, func(t *testing.T) { |
| 321 | f := newFixture(t) |
| 322 | pub, key, err := ed25519.GenerateKey(rand.Reader) |
| 323 | if err != nil { |
| 324 | t.Fatal(err) |
| 325 | } |
| 326 | if phase != "absent" && phase != "read-error" { |
| 327 | p := BackfillPlan{V: 1, ID: "backfill", Phase: phase, CreatedAt: f.now, Hosts: map[string]BackfillHost{}} |
| 328 | raw, err := release.SignRootEnvelope(release.RootSigner(key), p) |
| 329 | if err != nil { |
| 330 | t.Fatal(err) |
| 331 | } |
| 332 | if _, err = f.store.Put(f.ctx, BackfillKey, raw, ""); err != nil { |
| 333 | t.Fatal(err) |
| 334 | } |
| 335 | } |
| 336 | if phase == "bad-signature" { |
| 337 | pub = make(ed25519.PublicKey, ed25519.PublicKeySize) |
| 338 | } |
| 339 | if phase == "read-error" { |
| 340 | f.store.failGet = BackfillKey |
| 341 | } |
| 342 | c := NewBackfillController(f.store, hostactors.Directory{Store: f.store, PublicKey: pub, Bucket: "bucket"}, f.c.Now, f.c.NewID) |
| 343 | lists, adopts := 0, 0 |
| 344 | f.store.afterList = func(string, []string) error { lists++; return nil } |
| 345 | err = ReconcileClaims(f.ctx, c, func(context.Context, string) (hostclaim.Observed, error) { adopts++; return hostclaim.Observed{}, nil }) |
| 346 | wantError := phase == "adopting" || phase == "bad-signature" || phase == "read-error" |
| 347 | if (err != nil) != wantError || lists != 0 || adopts != 0 { |
| 348 | t.Fatalf("phase %s: err=%v lists=%d adopts=%d", phase, err, lists, adopts) |
| 349 | } |
| 350 | if !wantError && phase != "absent" { |
| 351 | if _, _, _, _, err := c.Authority.Adoption(f.ctx, Record{}); err == nil { |
| 352 | t.Fatal("inactive plan granted direct adoption") |
| 353 | } |
| 354 | } |
| 355 | }) |
| 356 | } |
| 357 | } |
| 358 | |
amadan