| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "crypto/ed25519" |
| 6 | "crypto/sha256" |
| 7 | "encoding/base64" |
| 8 | "encoding/hex" |
| 9 | "encoding/json" |
| 10 | "errors" |
| 11 | "fmt" |
| 12 | "os" |
| 13 | "slices" |
| 14 | "testing" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/hostclaim" |
| 18 | "github.com/carlosframework/platform/internal/release" |
| 19 | ) |
| 20 | |
| 21 | const ( |
| 22 | doneOne = "one.shop.example.com" |
| 23 | doneTwo = "two.shop.example.com" |
| 24 | ) |
| 25 | |
| 26 | // doneOtherKey signs the plan a wrong signer wrote. Fixed, so a mutation |
| 27 | // check can name it. |
| 28 | var doneOtherKey = ed25519.NewKeyFromSeed(bytes.Repeat([]byte{9}, ed25519.SeedSize)) |
| 29 | |
| 30 | // keyless re-signs the roster with these boxes' census keys removed: the dev |
| 31 | // roster's shape, where sophie has no key. |
| 32 | func (a *activation) keyless(refs ...string) { |
| 33 | a.t.Helper() |
| 34 | for _, ref := range refs { |
| 35 | box := a.roster.Boxes[ref] |
| 36 | box.CensusPublicKey = nil |
| 37 | a.roster.Boxes[ref] = box |
| 38 | } |
| 39 | _, tag, err := a.f.store.Get(a.ctx, "control/host-actors.json") |
| 40 | if err != nil { |
| 41 | a.t.Fatal(err) |
| 42 | } |
| 43 | raw, err := release.SignRootEnvelope(a.sg, a.roster) |
| 44 | if err != nil { |
| 45 | a.t.Fatal(err) |
| 46 | } |
| 47 | if _, err := a.f.store.Put(a.ctx, "control/host-actors.json", raw, tag); err != nil { |
| 48 | a.t.Fatal(err) |
| 49 | } |
| 50 | } |
| 51 | |
| 52 | // collecting writes backfill-collect's plan over the records (no box, no |
| 53 | // region), edited by edit when it is not nil, signed by signer. |
| 54 | func (a *activation) collecting(signer release.Signer, edit func(*BackfillPlan)) string { |
| 55 | a.t.Helper() |
| 56 | p := BackfillPlan{V: 1, ID: "backfill", Phase: "collecting", DirectoryRevision: 1, CreatedAt: a.f.now, Hosts: map[string]BackfillHost{}} |
| 57 | for _, r := range a.records { |
| 58 | p.Hosts[r.Holder.Host] = BackfillHost{Key: r.Holder.Key, ETag: r.ETag, Acquisition: r.Holder.Acquisition} |
| 59 | } |
| 60 | if edit != nil { |
| 61 | edit(&p) |
| 62 | } |
| 63 | raw, err := release.SignRootEnvelope(signer, p) |
| 64 | if err != nil { |
| 65 | a.t.Fatal(err) |
| 66 | } |
| 67 | _, before, err := a.f.store.Get(a.ctx, BackfillKey) |
| 68 | if err != nil && !errors.Is(err, os.ErrNotExist) { |
| 69 | a.t.Fatal(err) |
| 70 | } |
| 71 | tag, err := a.f.store.Put(a.ctx, BackfillKey, raw, before) |
| 72 | if err != nil { |
| 73 | a.t.Fatal(err) |
| 74 | } |
| 75 | return tag |
| 76 | } |
| 77 | |
| 78 | // emptyReport is a box's complete census report for the reservation, as |
| 79 | // the migration census writes it on an empty box. |
| 80 | func (a *activation) emptyReport(ref string) CensusReport { |
| 81 | return CensusReport{Box: ref, Enrollment: a.roster.Boxes[ref].Enrollment, BootID: "boot", Reservation: "backfill", DirectoryRevision: 1, |
| 82 | CapturedAt: a.f.now, Complete: true, Installations: []CensusInstallation{}, Seeds: &CensusSeeds{Journaled: []string{}, Gated: []string{}}} |
| 83 | } |
| 84 | |
| 85 | // rawReport signs report with key and writes it at the box's report key, |
| 86 | // bypassing PublishCensusReport's own checks, as a broken box could. |
| 87 | func (a *activation) rawReport(report CensusReport, key ed25519.PrivateKey) { |
| 88 | a.t.Helper() |
| 89 | canonical, err := json.Marshal(report) |
| 90 | if err != nil { |
| 91 | a.t.Fatal(err) |
| 92 | } |
| 93 | raw, err := json.Marshal(CensusEnvelope{Report: report, Signature: ed25519.Sign(key, canonical)}) |
| 94 | if err != nil { |
| 95 | a.t.Fatal(err) |
| 96 | } |
| 97 | k, err := reportKey("backfill", report.Box) |
| 98 | if err != nil { |
| 99 | a.t.Fatal(err) |
| 100 | } |
| 101 | if _, err := a.f.store.Put(a.ctx, k, raw, ""); err != nil { |
| 102 | a.t.Fatal(err) |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | func (a *activation) installation(host string) CensusInstallation { |
| 107 | a.t.Helper() |
| 108 | for _, r := range a.records { |
| 109 | if r.Holder.Host == host { |
| 110 | inc, err := r.Incarnation() |
| 111 | if err != nil { |
| 112 | a.t.Fatal(err) |
| 113 | } |
| 114 | return CensusInstallation{Host: host, Account: r.Holder.Account, App: r.Holder.App, Family: r.Holder.Family, Key: r.Holder.Key, Incarnation: inc, Route: true} |
| 115 | } |
| 116 | } |
| 117 | a.t.Fatalf("no record for %s", host) |
| 118 | return CensusInstallation{} |
| 119 | } |
| 120 | |
| 121 | // The empty cohort: two records frozen, box-a (keyed) reports nothing, |
| 122 | // box-b has no census key. Done writes done under If-Match with box-a's |
| 123 | // digest only, returns box-b as unreported, and lifts the create freeze. |
| 124 | // The console's adoption pass is a no-op on a done plan, adoption refuses |
| 125 | // it, and a second done is refused. |
| 126 | func TestBackfillDoneClosesAnEmptyCohort(t *testing.T) { |
| 127 | a := newActivation(t) |
| 128 | a.record(doneOne, nil) |
| 129 | a.record(doneTwo, nil) |
| 130 | a.keyless("box-b") |
| 131 | tag := a.collecting(a.sg, nil) |
| 132 | if err := CheckCreateFreeze(a.ctx, a.f.store, a.d.PublicKey); !errors.Is(err, ErrPending) { |
| 133 | t.Fatalf("freeze while collecting = %v", err) |
| 134 | } |
| 135 | if err := PublishCensusReport(a.ctx, a.f.store, a.d, a.emptyReport("box-a"), a.boxKeys["box-a"]); err != nil { |
| 136 | t.Fatal(err) |
| 137 | } |
| 138 | before := a.plan() |
| 139 | b := &Backfill{Store: a.f.store, Directory: a.d} |
| 140 | unreported, err := b.Done(a.ctx, a.sg, tag) |
| 141 | if err != nil || !slices.Equal(unreported, []string{"box-b"}) { |
| 142 | t.Fatalf("Done = %v, %v; want box-b unreported", unreported, err) |
| 143 | } |
| 144 | p := a.plan() |
| 145 | key, _ := reportKey("backfill", "box-a") |
| 146 | body, _, err := a.f.store.Get(a.ctx, key) |
| 147 | if err != nil { |
| 148 | t.Fatal(err) |
| 149 | } |
| 150 | sum := sha256.Sum256(body) |
| 151 | if p.Phase != "done" || len(p.Reports) != 1 || p.Reports["box-a"] != hex.EncodeToString(sum[:]) || p.ID != before.ID || !p.CreatedAt.Equal(before.CreatedAt) || p.DirectoryRevision != before.DirectoryRevision { |
| 152 | t.Fatalf("done plan = %+v", p) |
| 153 | } |
| 154 | for host, h := range before.Hosts { |
| 155 | if p.Hosts[host] != h { |
| 156 | t.Fatalf("%s changed: %+v -> %+v", host, h, p.Hosts[host]) |
| 157 | } |
| 158 | } |
| 159 | if len(p.Hosts) != 2 { |
| 160 | t.Fatalf("hosts = %+v", p.Hosts) |
| 161 | } |
| 162 | if err := CheckCreateFreeze(a.ctx, a.f.store, a.d.PublicKey); err != nil { |
| 163 | t.Fatalf("freeze after done = %v", err) |
| 164 | } |
| 165 | serial := 0 |
| 166 | c := NewBackfillController(a.f.store, a.d, func() time.Time { return a.f.now }, func() string { serial++; return fmt.Sprintf("op-%d", serial) }) |
| 167 | if err := c.ReconcileClaims(a.ctx); err != nil { |
| 168 | t.Fatalf("adoption pass over a done plan = %v", err) |
| 169 | } |
| 170 | for _, host := range []string{doneOne, doneTwo} { |
| 171 | if obs, err := hostclaim.Read(a.ctx, a.f.store, host); err != nil || !obs.Absent { |
| 172 | t.Fatalf("%s gained a claim: %+v %v", host, obs.Claim, err) |
| 173 | } |
| 174 | } |
| 175 | if _, _, _, _, err := b.Adoption(a.ctx, a.records[0]); !errors.Is(err, ErrContract) { |
| 176 | t.Fatalf("Adoption on a done plan = %v", err) |
| 177 | } |
| 178 | _, doneTag, err := a.f.store.Get(a.ctx, BackfillKey) |
| 179 | if err != nil { |
| 180 | t.Fatal(err) |
| 181 | } |
| 182 | if _, err := b.Done(a.ctx, a.sg, doneTag); !errors.Is(err, ErrDonePhase) { |
| 183 | t.Fatalf("second Done = %v", err) |
| 184 | } |
| 185 | } |
| 186 | |
| 187 | // The dev trial's real shape: every instance record deleted before the |
| 188 | // window, so the collecting plan freezes zero hosts. Its hosts are an empty |
| 189 | // object, never null, and Done accepts an empty listing against it. |
| 190 | func TestBackfillDoneWithZeroHosts(t *testing.T) { |
| 191 | a := newActivation(t) |
| 192 | a.keyless("box-b") |
| 193 | tag := a.collecting(a.sg, nil) |
| 194 | raw, _, err := a.f.store.Get(a.ctx, BackfillKey) |
| 195 | if err != nil { |
| 196 | t.Fatal(err) |
| 197 | } |
| 198 | var wire struct { |
| 199 | Payload string `json:"payload"` |
| 200 | } |
| 201 | if err := json.Unmarshal(raw, &wire); err != nil { |
| 202 | t.Fatal(err) |
| 203 | } |
| 204 | payload, err := base64.StdEncoding.DecodeString(wire.Payload) |
| 205 | if err != nil || !bytes.Contains(payload, []byte(`"hosts":{}`)) { |
| 206 | t.Fatalf("zero-host plan payload = %s, %v; want \"hosts\":{}", payload, err) |
| 207 | } |
| 208 | if p := a.plan(); p.Hosts == nil || len(p.Hosts) != 0 { |
| 209 | t.Fatalf("zero-host plan hosts = %#v", p.Hosts) |
| 210 | } |
| 211 | if err := PublishCensusReport(a.ctx, a.f.store, a.d, a.emptyReport("box-a"), a.boxKeys["box-a"]); err != nil { |
| 212 | t.Fatal(err) |
| 213 | } |
| 214 | b := &Backfill{Store: a.f.store, Directory: a.d} |
| 215 | unreported, err := b.Done(a.ctx, a.sg, tag) |
| 216 | if err != nil || !slices.Equal(unreported, []string{"box-b"}) { |
| 217 | t.Fatalf("Done over zero hosts = %v, %v", unreported, err) |
| 218 | } |
| 219 | if p := a.plan(); p.Phase != "done" || p.Hosts == nil || len(p.Hosts) != 0 || len(p.Reports) != 1 { |
| 220 | t.Fatalf("done plan = %#v", p) |
| 221 | } |
| 222 | if err := CheckCreateFreeze(a.ctx, a.f.store, a.d.PublicKey); err != nil { |
| 223 | t.Fatal(err) |
| 224 | } |
| 225 | // A record that appears after a zero-host collect is refused. |
| 226 | b2 := newActivation(t) |
| 227 | b2.keyless("box-b") |
| 228 | tag = b2.collecting(b2.sg, nil) |
| 229 | if err := PublishCensusReport(b2.ctx, b2.f.store, b2.d, b2.emptyReport("box-a"), b2.boxKeys["box-a"]); err != nil { |
| 230 | t.Fatal(err) |
| 231 | } |
| 232 | b2.record(doneOne, nil) |
| 233 | if _, err := (&Backfill{Store: b2.f.store, Directory: b2.d}).Done(b2.ctx, b2.sg, tag); !errors.Is(err, ErrDoneRecords) { |
| 234 | t.Fatalf("Done with a record the zero-host plan does not name = %v", err) |
| 235 | } |
| 236 | } |
| 237 | |
| 238 | // Every condition is its own refusal, and a refusal writes nothing: the |
| 239 | // plan stays collecting, byte for byte, and creates stay frozen. |
| 240 | func TestBackfillDoneRefusals(t *testing.T) { |
| 241 | type setup struct { |
| 242 | keyless []string // boxes with no census key (default box-b) |
| 243 | edit func(*BackfillPlan) // the collecting plan |
| 244 | signer bool // sign the plan with doneOtherKey |
| 245 | reports func(a *activation) // default: box-a's empty report |
| 246 | after func(a *activation) // changes after the reports |
| 247 | tag func(t string) string // the etag Done is given |
| 248 | } |
| 249 | for name, c := range map[string]struct { |
| 250 | s setup |
| 251 | want []error |
| 252 | }{ |
| 253 | "etag": {setup{tag: func(string) string { return "other" }}, []error{ErrDoneETag}}, |
| 254 | "envelope": {setup{signer: true}, []error{ErrDoneEnvelope}}, |
| 255 | "phase": {setup{edit: func(p *BackfillPlan) { p.Phase = "adopting"; p.Reports = map[string]string{} }}, []error{ErrDonePhase}}, |
| 256 | "record etag": {setup{after: func(a *activation) { |
| 257 | r := a.records[0] |
| 258 | if _, err := a.f.store.Put(a.ctx, r.Holder.Key, append(append([]byte(nil), r.Body...), '\n'), r.ETag); err != nil { |
| 259 | a.t.Fatal(err) |
| 260 | } |
| 261 | }}, []error{ErrDoneRecords}}, |
| 262 | "record added": {setup{after: func(a *activation) { a.record("three.shop.example.com", nil) }}, []error{ErrDoneRecords}}, |
| 263 | "record gone": {setup{after: func(a *activation) { |
| 264 | if err := release.DeleteIfMatch(a.ctx, a.f.store, a.records[1].Holder.Key, a.records[1].ETag); err != nil { |
| 265 | a.t.Fatal(err) |
| 266 | } |
| 267 | }}, []error{ErrDoneRecords}}, |
| 268 | "acquisition": {setup{edit: func(p *BackfillPlan) { |
| 269 | h := p.Hosts[doneOne] |
| 270 | h.Acquisition = "other" |
| 271 | p.Hosts[doneOne] = h |
| 272 | }}, []error{ErrDoneRecords}}, |
| 273 | "claim": {setup{after: func(a *activation) { |
| 274 | key, err := hostclaim.Key(doneOne) |
| 275 | if err != nil { |
| 276 | a.t.Fatal(err) |
| 277 | } |
| 278 | if _, err := a.f.store.Put(a.ctx, key, []byte(`{}`), ""); err != nil { |
| 279 | a.t.Fatal(err) |
| 280 | } |
| 281 | }}, []error{ErrDoneClaims}}, |
| 282 | "admission": {setup{after: func(a *activation) { |
| 283 | key, err := AdmissionKey("bab", "shop") |
| 284 | if err != nil { |
| 285 | a.t.Fatal(err) |
| 286 | } |
| 287 | if _, err := a.f.store.Put(a.ctx, key, []byte(`{}`), ""); err != nil { |
| 288 | a.t.Fatal(err) |
| 289 | } |
| 290 | }}, []error{ErrDoneAdmission}}, |
| 291 | "no report": {setup{reports: func(*activation) {}}, []error{ErrDoneReport, os.ErrNotExist}}, |
| 292 | "keyed box without a report": {setup{keyless: []string{}}, []error{ErrDoneReport, os.ErrNotExist}}, |
| 293 | "no keyed box": {setup{keyless: []string{"box-a", "box-b"}, reports: func(*activation) {}}, []error{ErrDoneReport}}, |
| 294 | "report signature": {setup{reports: func(a *activation) { a.rawReport(a.emptyReport("box-a"), a.boxKeys["box-b"]) }}, []error{ErrDoneReport}}, |
| 295 | "report revision": {setup{reports: func(a *activation) { |
| 296 | r := a.emptyReport("box-a") |
| 297 | r.DirectoryRevision = 2 |
| 298 | a.rawReport(r, a.boxKeys["box-a"]) |
| 299 | }}, []error{ErrDoneReport}}, |
| 300 | "report reservation": {setup{reports: func(a *activation) { |
| 301 | r := a.emptyReport("box-a") |
| 302 | r.Reservation = "other" |
| 303 | a.rawReport(r, a.boxKeys["box-a"]) |
| 304 | }}, []error{ErrDoneReport}}, |
| 305 | "report incomplete": {setup{reports: func(a *activation) { |
| 306 | r := a.emptyReport("box-a") |
| 307 | r.Complete = false |
| 308 | a.rawReport(r, a.boxKeys["box-a"]) |
| 309 | }}, []error{ErrDoneReport}}, |
| 310 | "report captured before the plan": {setup{reports: func(a *activation) { |
| 311 | r := a.emptyReport("box-a") |
| 312 | r.CapturedAt = a.f.now.Add(-time.Second) |
| 313 | if err := PublishCensusReport(a.ctx, a.f.store, a.d, r, a.boxKeys["box-a"]); err != nil { |
| 314 | a.t.Fatal(err) |
| 315 | } |
| 316 | }}, []error{ErrDoneStale}}, |
| 317 | "installations": {setup{reports: func(a *activation) { |
| 318 | r := a.emptyReport("box-a") |
| 319 | r.Installations = []CensusInstallation{a.installation(doneOne)} |
| 320 | if err := PublishCensusReport(a.ctx, a.f.store, a.d, r, a.boxKeys["box-a"]); err != nil { |
| 321 | a.t.Fatal(err) |
| 322 | } |
| 323 | }}, []error{ErrDoneInstallations}}, |
| 324 | "roster moved": {setup{after: func(a *activation) { |
| 325 | a.roster.Revision = 2 |
| 326 | a.keyless() |
| 327 | }}, []error{ErrPending}}, |
| 328 | } { |
| 329 | t.Run(name, func(t *testing.T) { |
| 330 | a := newActivation(t) |
| 331 | a.record(doneOne, nil) |
| 332 | a.record(doneTwo, nil) |
| 333 | keyless := []string{"box-b"} |
| 334 | if c.s.keyless != nil { |
| 335 | keyless = c.s.keyless |
| 336 | } |
| 337 | a.keyless(keyless...) |
| 338 | signer := a.sg |
| 339 | if c.s.signer { |
| 340 | signer = release.RootSigner(doneOtherKey) |
| 341 | } |
| 342 | tag := a.collecting(signer, c.s.edit) |
| 343 | if c.s.reports != nil { |
| 344 | c.s.reports(a) |
| 345 | } else if err := PublishCensusReport(a.ctx, a.f.store, a.d, a.emptyReport("box-a"), a.boxKeys["box-a"]); err != nil { |
| 346 | t.Fatal(err) |
| 347 | } |
| 348 | if c.s.after != nil { |
| 349 | c.s.after(a) |
| 350 | } |
| 351 | if c.s.tag != nil { |
| 352 | tag = c.s.tag(tag) |
| 353 | } |
| 354 | before, beforeTag, err := a.f.store.Get(a.ctx, BackfillKey) |
| 355 | if err != nil { |
| 356 | t.Fatal(err) |
| 357 | } |
| 358 | b := &Backfill{Store: a.f.store, Directory: a.d} |
| 359 | unreported, err := b.Done(a.ctx, a.sg, tag) |
| 360 | for _, want := range c.want { |
| 361 | if !errors.Is(err, want) { |
| 362 | t.Fatalf("Done = %v, %v; want %v", unreported, err, want) |
| 363 | } |
| 364 | } |
| 365 | after, afterTag, err := a.f.store.Get(a.ctx, BackfillKey) |
| 366 | if err != nil || !bytes.Equal(before, after) || beforeTag != afterTag { |
| 367 | t.Fatalf("a refused Done changed the plan (%v)", err) |
| 368 | } |
| 369 | if err := CheckCreateFreeze(a.ctx, a.f.store, a.d.PublicKey); err == nil { |
| 370 | t.Fatal("a refused Done lifted the create freeze") |
| 371 | } |
| 372 | }) |
| 373 | } |
| 374 | } |
| 375 | |
| 376 | // Done's copy of the census's per-box report check must accept and refuse |
| 377 | // exactly what ProductionAuthority.census does (census is accepted-legacy |
| 378 | // and stays untouched). Each variant is box-a's report; box-b's is valid. |
| 379 | func TestDoneVerifiesReportsAsTheCensusDoes(t *testing.T) { |
| 380 | for name, c := range map[string]struct { |
| 381 | edit func(a *activation, r *CensusReport) |
| 382 | key string // signing box key (default box-a) |
| 383 | }{ |
| 384 | "valid": {}, |
| 385 | "valid with a host": {edit: func(a *activation, r *CensusReport) { r.Installations = []CensusInstallation{a.installation(doneOne)} }}, |
| 386 | "other box's key": {key: "box-b"}, |
| 387 | "other box": {edit: func(_ *activation, r *CensusReport) { r.Box = "box-b" }}, |
| 388 | "other enrollment": {edit: func(_ *activation, r *CensusReport) { r.Enrollment = "x" }}, |
| 389 | "other reservation": {edit: func(_ *activation, r *CensusReport) { r.Reservation = "other" }}, |
| 390 | "other revision": {edit: func(_ *activation, r *CensusReport) { r.DirectoryRevision = 2 }}, |
| 391 | "no boot": {edit: func(_ *activation, r *CensusReport) { r.BootID = "" }}, |
| 392 | "no capture time": {edit: func(_ *activation, r *CensusReport) { r.CapturedAt = time.Time{} }}, |
| 393 | "nil installations": {edit: func(_ *activation, r *CensusReport) { r.Installations = nil }}, |
| 394 | "incomplete": {edit: func(_ *activation, r *CensusReport) { r.Complete = false }}, |
| 395 | "unknown host": {edit: func(a *activation, r *CensusReport) { |
| 396 | i := a.installation(doneOne) |
| 397 | i.Host = "x.example.com" |
| 398 | r.Installations = []CensusInstallation{i} |
| 399 | }}, |
| 400 | "host twice": {edit: func(a *activation, r *CensusReport) { |
| 401 | i := a.installation(doneOne) |
| 402 | r.Installations = []CensusInstallation{i, i} |
| 403 | }}, |
| 404 | "neither route nor local": {edit: func(a *activation, r *CensusReport) { |
| 405 | i := a.installation(doneOne) |
| 406 | i.Route = false |
| 407 | r.Installations = []CensusInstallation{i} |
| 408 | }}, |
| 409 | "other account": {edit: func(a *activation, r *CensusReport) { |
| 410 | i := a.installation(doneOne) |
| 411 | i.Account = "zzz" |
| 412 | r.Installations = []CensusInstallation{i} |
| 413 | }}, |
| 414 | "other key": {edit: func(a *activation, r *CensusReport) { |
| 415 | i := a.installation(doneOne) |
| 416 | i.Key = "apps/x" |
| 417 | r.Installations = []CensusInstallation{i} |
| 418 | }}, |
| 419 | "other family": {edit: func(a *activation, r *CensusReport) { |
| 420 | i := a.installation(doneOne) |
| 421 | i.Family = "sidecar" |
| 422 | r.Installations = []CensusInstallation{i} |
| 423 | }}, |
| 424 | "other incarnation": {edit: func(a *activation, r *CensusReport) { |
| 425 | i := a.installation(doneOne) |
| 426 | i.Incarnation = "x" |
| 427 | r.Installations = []CensusInstallation{i} |
| 428 | }}, |
| 429 | } { |
| 430 | t.Run(name, func(t *testing.T) { |
| 431 | a := newActivation(t) |
| 432 | a.record(doneOne, nil) |
| 433 | a.collecting(a.sg, nil) |
| 434 | if err := PublishCensusReport(a.ctx, a.f.store, a.d, a.emptyReport("box-b"), a.boxKeys["box-b"]); err != nil { |
| 435 | t.Fatal(err) |
| 436 | } |
| 437 | r := a.emptyReport("box-a") |
| 438 | if c.edit != nil { |
| 439 | c.edit(a, &r) |
| 440 | } |
| 441 | key := a.boxKeys["box-a"] |
| 442 | if c.key != "" { |
| 443 | key = a.boxKeys[c.key] |
| 444 | } |
| 445 | k, _ := reportKey("backfill", "box-a") |
| 446 | canonical, _ := json.Marshal(r) |
| 447 | raw, _ := json.Marshal(CensusEnvelope{Report: r, Signature: ed25519.Sign(key, canonical)}) |
| 448 | tag, err := a.f.store.Put(a.ctx, k, raw, "") |
| 449 | if err != nil { |
| 450 | t.Fatal(err) |
| 451 | } |
| 452 | _, censusErr := ProductionAuthority{Store: a.f.store, Directory: a.d}.census(a.ctx, "", "", "backfill", 0, nil) |
| 453 | records := map[string]Record{} |
| 454 | for _, rec := range a.records { |
| 455 | records[rec.Holder.Host] = rec |
| 456 | } |
| 457 | _, doneErr := verifyCensusReport(raw, tag, "box-a", a.roster.Boxes["box-a"], "backfill", a.roster.Revision, records) |
| 458 | if (censusErr == nil) != (doneErr == nil) { |
| 459 | t.Fatalf("census = %v, Done's check = %v", censusErr, doneErr) |
| 460 | } |
| 461 | if name == "valid" || name == "valid with a host" { |
| 462 | if doneErr != nil { |
| 463 | t.Fatalf("valid report refused: %v", doneErr) |
| 464 | } |
| 465 | } else if doneErr == nil { |
| 466 | t.Fatal("variant accepted") |
| 467 | } |
| 468 | }) |
| 469 | } |
| 470 | } |
| 471 | |
| 472 | // doneCohort is the done tests' empty cohort: hosts records (zero or two), |
| 473 | // box-b keyless, box-a's empty report published, the collecting plan's etag |
| 474 | // returned. |
| 475 | func doneCohort(t *testing.T, hosts int) (*activation, string) { |
| 476 | t.Helper() |
| 477 | a := newActivation(t) |
| 478 | for _, host := range []string{doneOne, doneTwo}[:hosts] { |
| 479 | a.record(host, nil) |
| 480 | } |
| 481 | a.keyless("box-b") |
| 482 | tag := a.collecting(a.sg, nil) |
| 483 | if err := PublishCensusReport(a.ctx, a.f.store, a.d, a.emptyReport("box-a"), a.boxKeys["box-a"]); err != nil { |
| 484 | t.Fatal(err) |
| 485 | } |
| 486 | return a, tag |
| 487 | } |
| 488 | |
| 489 | // A record created while Done reads the reports (the first listing has |
| 490 | // already passed) is caught by the second listing, immediately before the |
| 491 | // write: Done refuses and writes nothing. |
| 492 | func TestBackfillDoneRefusesARecordCreatedDuringReportRetrieval(t *testing.T) { |
| 493 | for _, hosts := range []int{0, 2} { |
| 494 | t.Run(fmt.Sprintf("%d hosts", hosts), func(t *testing.T) { |
| 495 | a, tag := doneCohort(t, hosts) |
| 496 | reportAt, err := reportKey("backfill", "box-a") |
| 497 | if err != nil { |
| 498 | t.Fatal(err) |
| 499 | } |
| 500 | fired := false |
| 501 | a.f.store.afterGet = func(key string) { |
| 502 | if key != reportAt { |
| 503 | return |
| 504 | } |
| 505 | a.f.store.afterGet = nil |
| 506 | fired = true |
| 507 | a.record("three.shop.example.com", nil) |
| 508 | } |
| 509 | before, beforeTag, err := a.f.store.Get(a.ctx, BackfillKey) |
| 510 | if err != nil { |
| 511 | t.Fatal(err) |
| 512 | } |
| 513 | _, err = (&Backfill{Store: a.f.store, Directory: a.d}).Done(a.ctx, a.sg, tag) |
| 514 | if !fired { |
| 515 | t.Fatal("the report was never read") |
| 516 | } |
| 517 | if !errors.Is(err, ErrDoneRecords) || errors.Is(err, ErrDoneRecordsChanged) { |
| 518 | t.Fatalf("Done = %v; want the records refusal before the write", err) |
| 519 | } |
| 520 | after, afterTag, err := a.f.store.Get(a.ctx, BackfillKey) |
| 521 | if err != nil || !bytes.Equal(before, after) || beforeTag != afterTag { |
| 522 | t.Fatalf("a refused Done changed the plan (%v)", err) |
| 523 | } |
| 524 | if err := CheckCreateFreeze(a.ctx, a.f.store, a.d.PublicKey); !errors.Is(err, ErrPending) { |
| 525 | t.Fatalf("freeze after a refused Done = %v", err) |
| 526 | } |
| 527 | }) |
| 528 | } |
| 529 | } |
| 530 | |
| 531 | // A record created after Done's write, before its third listing, is not in |
| 532 | // the done plan. Done has written done and cannot take it back; it says |
| 533 | // so with ErrDoneRecordsChanged (which the claims run reports under the |
| 534 | // records refusal). |
| 535 | func TestBackfillDoneReportsARecordCreatedAfterTheWrite(t *testing.T) { |
| 536 | for _, hosts := range []int{0, 2} { |
| 537 | t.Run(fmt.Sprintf("%d hosts", hosts), func(t *testing.T) { |
| 538 | a, tag := doneCohort(t, hosts) |
| 539 | fired := false |
| 540 | a.f.store.afterPut = func(key string) { |
| 541 | if key != BackfillKey { |
| 542 | return |
| 543 | } |
| 544 | a.f.store.afterPut = nil |
| 545 | fired = true |
| 546 | a.record("three.shop.example.com", nil) |
| 547 | } |
| 548 | _, err := (&Backfill{Store: a.f.store, Directory: a.d}).Done(a.ctx, a.sg, tag) |
| 549 | if !fired { |
| 550 | t.Fatal("Done never wrote the plan") |
| 551 | } |
| 552 | if !errors.Is(err, ErrDoneRecordsChanged) || !errors.Is(err, ErrDoneRecords) { |
| 553 | t.Fatalf("Done = %v; want ErrDoneRecordsChanged", err) |
| 554 | } |
| 555 | if p := a.plan(); p.Phase != "done" || len(p.Hosts) != hosts { |
| 556 | t.Fatalf("plan = %+v; want done over the %d frozen hosts", p, hosts) |
| 557 | } |
| 558 | }) |
| 559 | } |
| 560 | } |
| 561 | |
amadan