amadan

carlos / platform Public

console: eligible boxes come from the roster's proven cloud, and a tie breaks by the host's hash; activator: no local copy is trusted

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude ab345ada1e36a14cdbd32f2882d2f8a319a37848 parent 9be3824
5 files changed, +136 −122
  • internal/activator/activator.go +5 −1
  • internal/activator/installed.go +5 −34
  • internal/activator/installed_test.go +0 −32
  • internal/console/boxchoice.go +50 −26
  • internal/console/boxchoice_test.go +76 −29
diff --git a/internal/activator/activator.go b/internal/activator/activator.go
index a1afa63f4..daf3f7012 100644
--- a/internal/activator/activator.go
+++ b/internal/activator/activator.go
@@ -1165,7 +1165,11 @@ func (a *Service) doWake(ctx context.Context, e *actEntry) (*wokenInstance, erro
// happen *under* the lock (restoring without it could later shadow newer
// S3 state written by the true holder). When it's still on disk it is the
// warm cache: on one box, with the lease held, the local copy can only be
- // what we ourselves checkpointed at the last sleep.
+ // what we ourselves checkpointed at the last sleep. That holds only while
+ // a region has one box. Claim ruling R6 ends it (region HA spec 1 H13):
+ // once wakes are admitted by hostclaim's AdmitWriter, every admitted
+ // writer purges its local copy and restores from the certified remote,
+ // and no marker is ever read to skip that.
fresh, err := a.backend.Restore(ctx, rt)
if err != nil {
w.stopOutcome.leaseReleased = lease.Release(ctx) == nil
diff --git a/internal/activator/installed.go b/internal/activator/installed.go
index 8ebdd6b3d..60d6c7e40 100644
--- a/internal/activator/installed.go
+++ b/internal/activator/installed.go
@@ -10,14 +10,11 @@ import (
"github.com/carlosframework/platform/internal/provision"
)
-// InstallMarker is a box's own record that it finished installing one
-// generation of a host: the database restored and synced, the files
-// applied, the wake state imported (region HA spec 1 §5.5). It is written
-// last, so its presence vouches for all of them.
-//
-// It is evidence, not authority. Authority is the claim naming this box,
-// which the box holds as its active stamp. A box that crashed mid-install
-// has authority and no marker, and must restore.
+// InstallMarker is a box's own note that it finished a local install of
+// one generation of a host. Nothing reads it to trust a local database:
+// claim ruling R6 has every admitted writer purge and restore from the
+// certified remote (region HA spec 1 H13, §5.5 withdrawn). It is kept for
+// relocation's local install journal (§5.8), which plan 1b does not build.
type InstallMarker struct {
Incarnation string `json:"incarnation"`
Generation string `json:"generation"`
@@ -26,13 +23,6 @@ type InstallMarker struct {
FilesSeq int64 `json:"files_seq"`
}
-// Installation is what the claim says is current for a host.
-type Installation struct {
- Incarnation string
- Generation string
- Stamp int64
-}
-
func InstallMarkerPath(dataDir, host string) string {
return filepath.Join(dataDir, ".installed", provision.InstanceID(host)+".json")
}
@@ -101,22 +91,3 @@ func RemoveInstallMarker(dataDir, host string) error {
}
return err
}
-
-// WarmCopyTrusted is invariant HA3: a box may skip restore and serve its
-// local copy only when it has authority for the current installation
-// (activeStamp equals the claim's stamp) AND evidence that it finished
-// installing exactly that one (the marker matches the claim, and the
-// database on disk is the one the marker was written for).
-//
-// dbMarker is the generation marker read from beside the local database.
-// Every empty value fails: nothing matches by being absent on both sides.
-func WarmCopyTrusted(want Installation, activeStamp int64, m InstallMarker, found bool, dbMarker string) bool {
- if !found || want.Stamp == 0 || want.Incarnation == "" || want.Generation == "" || dbMarker == "" {
- return false
- }
- return activeStamp == want.Stamp &&
- m.Stamp == want.Stamp &&
- m.Incarnation == want.Incarnation &&
- m.Generation == want.Generation &&
- m.DBMarker == dbMarker
-}
diff --git a/internal/activator/installed_test.go b/internal/activator/installed_test.go
index d6551421b..a18c8a9f2 100644
--- a/internal/activator/installed_test.go
+++ b/internal/activator/installed_test.go
@@ -72,35 +72,3 @@ func TestADamagedMarkerIsAnErrorNotAbsence(t *testing.T) {
t.Fatalf("found=%v err=%v: a damaged marker must be an error", found, err)
}
}
-
-func TestWarmCopyTrusted(t *testing.T) {
- want := Installation{Incarnation: "inc1", Generation: "g-op7", Stamp: 41}
- good := InstallMarker{Incarnation: "inc1", Generation: "g-op7", Stamp: 41, DBMarker: "dbm", FilesSeq: 9}
- if !WarmCopyTrusted(want, 41, good, true, "dbm") {
- t.Fatal("a complete, current installation was not trusted")
- }
- for name, tc := range map[string]struct {
- active int64
- m InstallMarker
- found bool
- db string
- }{
- "no marker: crashed before the install finished": {41, InstallMarker{}, false, "dbm"},
- "authority without evidence": {41, InstallMarker{Incarnation: "inc1", Generation: "g-op7", Stamp: 40, DBMarker: "dbm"}, true, "dbm"},
- "evidence without authority": {40, good, true, "dbm"},
- "no authority at all": {0, good, true, "dbm"},
- "another generation": {41, InstallMarker{Incarnation: "inc1", Generation: "g-op6", Stamp: 41, DBMarker: "dbm"}, true, "dbm"},
- "another incarnation": {41, InstallMarker{Incarnation: "inc0", Generation: "g-op7", Stamp: 41, DBMarker: "dbm"}, true, "dbm"},
- "the database is not the one installed": {41, good, true, "other"},
- "the database has no marker": {41, good, true, ""},
- } {
- t.Run(name, func(t *testing.T) {
- if WarmCopyTrusted(want, tc.active, tc.m, tc.found, tc.db) {
- t.Fatal("trusted")
- }
- })
- }
- if WarmCopyTrusted(Installation{}, 0, InstallMarker{}, true, "") {
- t.Fatal("an empty claim matched an empty marker")
- }
-}
diff --git a/internal/console/boxchoice.go b/internal/console/boxchoice.go
index 35f81de3b..32d8dea64 100644
--- a/internal/console/boxchoice.go
+++ b/internal/console/boxchoice.go
@@ -1,6 +1,8 @@
package console
import (
+ "crypto/sha256"
+ "encoding/hex"
"slices"
"sort"
@@ -16,44 +18,56 @@ type boxCandidate struct {
Awake int
}
+// rosterBox is what the roster (control/host-actors.json, claim D7) says
+// about one active box: the placement it is enrolled in and the cloud
+// identity enrollment proved from its boxid record (spec §5.1, H11). The
+// caller passes active entries only.
+type rosterBox struct {
+ Placement string
+ Cloud boxid.Cloud
+}
+
// eligibleBoxes is regionCandidates' predicate applied to one box at a time,
-// plus what a multi-box region adds: a verified box record that agrees with
-// the file's box_id and whose signed region is awsRegion (the AWS region
-// the caller expects for this placement), the HA capability, not draining,
-// not lost. Two boxes
-// that each satisfy half of it never add up to one.
+// plus what a multi-box region adds: a roster entry for the file's box_id,
+// enrolled in this placement, whose proven cloud is in awsRegion (the AWS
+// region the roster records for this placement), the HA capability, not
+// draining, not lost. Two boxes that each satisfy half of it never add up
+// to one. The edges file is evidence of liveness and configuration only:
+// identity and region come from the roster (spec §5.1).
//
// An incomplete snapshot offers nothing: an unread file could be the one
// box that would change the answer.
-func eligibleBoxes(snap edgesSnapshot, verified map[string]boxid.Identity, placement, awsRegion, homeName, expectedFP string, lost map[string]bool) []boxCandidate {
+func eligibleBoxes(snap edgesSnapshot, roster map[string]rosterBox, placement, awsRegion, homeName, expectedFP string, lost map[string]bool) []boxCandidate {
if !snap.complete() || expectedFP == "" || awsRegion == "" {
return nil
}
+ home := func(code string) string {
+ if homeName != "" && code == homeName {
+ return ""
+ }
+ return code
+ }
var out []boxCandidate
for _, f := range snap.Files {
- code := f.Placement
- if code == "" || (homeName != "" && code == homeName) {
- code = ""
- }
- want := placement
- if homeName != "" && want == homeName {
- want = ""
- }
- id, ok := verified[f.BoxID]
+ want := home(placement)
+ rb, ok := roster[f.BoxID]
switch {
- case code != want:
+ case home(f.Placement) != want:
case !f.Fresh, f.EIP == "", f.BoxID == "":
- case !ok || id.InstanceID == "":
- case id.Region != awsRegion, f.Region != awsRegion:
- // The placement code is the box's own word. The AWS region is
- // signed: a box whose identity is in another region is not in
- // this placement, whatever its file says.
+ case !ok || !rb.Cloud.Complete():
+ case home(rb.Placement) != want:
+ // The file's placement is the box's own word; the roster's is
+ // the enrollment's. Both must agree.
+ case rb.Cloud.Region != awsRegion, f.Region != awsRegion:
+ // The AWS region was proven at enrollment: a box enrolled in
+ // another region is not in this placement, whatever its file
+ // says.
case !isTrue(f.HibernationConfigured), !isTrue(f.ServesSharedPool), !isTrue(f.AcceptsInbound):
case f.SecretsKeyFP != expectedFP:
case !slices.Contains(f.Capabilities, "region-ha-1"):
case f.Draining, lost[f.BoxID]:
default:
- c := boxCandidate{BoxID: f.BoxID, Region: id.Region}
+ c := boxCandidate{BoxID: f.BoxID, Region: rb.Cloud.Region}
if f.Headroom != nil {
c.MemFreeMiB, c.DiskFreeMiB, c.Awake = f.Headroom.MemFreeMiB, f.Headroom.DiskFreeMiB, f.Headroom.Awake
}
@@ -70,13 +84,19 @@ type boxLoad struct {
Apps map[string]int // "acct/app" -> hosts of that app
}
-// chooseBox is spec 1 §5.2's interim rule until spec 2: prefer a box that
-// runs no instance of this app (soft spread: it never refuses), then the
-// fewest hosts overall, then the lowest box id.
-func chooseBox(cands []boxCandidate, running map[string]boxLoad, app string) (boxCandidate, bool) {
+// chooseBox is spec 1 §5.2's choose() until spec 2 (H12): prefer a box
+// that runs no instance of this app (soft spread: it never refuses), then
+// the fewest hosts overall, then the greatest hexadecimal SHA-256 of
+// host + "\x00" + box reference (claim P5-D2), then the smallest reference
+// if two hashes are equal.
+func chooseBox(host string, cands []boxCandidate, running map[string]boxLoad, app string) (boxCandidate, bool) {
if len(cands) == 0 {
return boxCandidate{}, false
}
+ tie := func(ref string) string {
+ sum := sha256.Sum256([]byte(host + "\x00" + ref))
+ return hex.EncodeToString(sum[:])
+ }
best := cands[0]
for _, c := range cands[1:] {
cl, bl := running[c.BoxID], running[best.BoxID]
@@ -90,6 +110,10 @@ func chooseBox(cands []boxCandidate, running map[string]boxLoad, app string) (bo
if cl.Hosts < bl.Hosts {
best = c
}
+ case tie(c.BoxID) != tie(best.BoxID):
+ if tie(c.BoxID) > tie(best.BoxID) {
+ best = c
+ }
case c.BoxID < best.BoxID:
best = c
}
diff --git a/internal/console/boxchoice_test.go b/internal/console/boxchoice_test.go
index 18713ecc6..b2aa98c6d 100644
--- a/internal/console/boxchoice_test.go
+++ b/internal/console/boxchoice_test.go
@@ -1,6 +1,8 @@
package console
import (
+ "crypto/sha256"
+ "encoding/hex"
"encoding/json"
"testing"
@@ -30,10 +32,11 @@ func ids(cs []boxCandidate) []string {
return out
}
-func verifiedFor(boxes ...string) map[string]boxid.Identity {
- out := map[string]boxid.Identity{}
+// rosterFor enrolls each box in placement "uk" as an instance in eu-west-2.
+func rosterFor(boxes ...string) map[string]rosterBox {
+ out := map[string]rosterBox{}
for _, b := range boxes {
- out[b] = boxid.Identity{AccountID: "1", Region: "eu-west-2", InstanceID: b}
+ out[b] = rosterBox{Placement: "uk", Cloud: boxid.Cloud{Provider: "aws", Account: "1", Region: "eu-west-2", InstanceID: "i-" + b}}
}
return out
}
@@ -56,12 +59,12 @@ func TestEligibleBoxesAppliesThePredicateOneBoxAtATime(t *testing.T) {
{"no capability", del("capabilities"), false},
{"draining", set("draining", true), false},
{"another placement", set("placement", "us"), false},
- {"box_id disagrees with its record", set("box_id", "i-0zzz"), false},
+ {"box_id names no roster entry", set("box_id", "i-0zzz"), false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
snap := views(t, haBox("i-0aaa", tc.mut))
- got := eligibleBoxes(snap, verifiedFor("i-0aaa"), "uk", "eu-west-2", "ie", testKeyFP, nil)
+ got := eligibleBoxes(snap, rosterFor("i-0aaa"), "uk", "eu-west-2", "ie", testKeyFP, nil)
if (len(got) == 1) != tc.want {
t.Fatalf("eligible = %v, want %v", ids(got), tc.want)
}
@@ -69,14 +72,14 @@ func TestEligibleBoxesAppliesThePredicateOneBoxAtATime(t *testing.T) {
}
}
-func TestEligibleBoxesNeedsAVerifiedRecordFreshnessAndNotLost(t *testing.T) {
+func TestEligibleBoxesNeedsARosterEntryFreshnessAndNotLost(t *testing.T) {
snap := views(t, haBox("i-0aaa", nil), haBox("i-0bbb", nil), haBox("i-0ccc", nil))
snap = markStale(snap, "i-0bbb")
- got := eligibleBoxes(snap, verifiedFor("i-0aaa", "i-0bbb"), "uk", "eu-west-2", "ie", testKeyFP, nil)
+ got := eligibleBoxes(snap, rosterFor("i-0aaa", "i-0bbb"), "uk", "eu-west-2", "ie", testKeyFP, nil)
if len(got) != 1 || got[0].BoxID != "i-0aaa" {
- t.Fatalf("got %v, want only i-0aaa (i-0bbb is stale, i-0ccc has no verified record)", ids(got))
+ t.Fatalf("got %v, want only i-0aaa (i-0bbb is stale, i-0ccc has no roster entry)", ids(got))
}
- got = eligibleBoxes(snap, verifiedFor("i-0aaa"), "uk", "eu-west-2", "ie", testKeyFP, map[string]bool{"i-0aaa": true})
+ got = eligibleBoxes(snap, rosterFor("i-0aaa"), "uk", "eu-west-2", "ie", testKeyFP, map[string]bool{"i-0aaa": true})
if len(got) != 0 {
t.Fatalf("a lost box is eligible: %v", ids(got))
}
@@ -88,7 +91,7 @@ func TestEligibleBoxesNeverCombinesTwoHalfBoxes(t *testing.T) {
snap := views(t,
haBox("i-0aaa", func(m map[string]any) { m["secrets_key_fp"] = "" }),
haBox("i-0bbb", func(m map[string]any) { m["accepts_inbound"] = false }))
- if got := eligibleBoxes(snap, verifiedFor("i-0aaa", "i-0bbb"), "uk", "eu-west-2", "ie", testKeyFP, nil); len(got) != 0 {
+ if got := eligibleBoxes(snap, rosterFor("i-0aaa", "i-0bbb"), "uk", "eu-west-2", "ie", testKeyFP, nil); len(got) != 0 {
t.Fatalf("got %v, want none", ids(got))
}
}
@@ -96,7 +99,7 @@ func TestEligibleBoxesNeverCombinesTwoHalfBoxes(t *testing.T) {
func TestEligibleBoxesAnIncompleteSnapshotOffersNothing(t *testing.T) {
snap := views(t, haBox("i-0aaa", nil))
snap.Complete = false
- if got := eligibleBoxes(snap, verifiedFor("i-0aaa"), "uk", "eu-west-2", "ie", testKeyFP, nil); len(got) != 0 {
+ if got := eligibleBoxes(snap, rosterFor("i-0aaa"), "uk", "eu-west-2", "ie", testKeyFP, nil); len(got) != 0 {
t.Fatalf("got %v from an incomplete snapshot", ids(got))
}
}
@@ -105,14 +108,20 @@ func TestEligibleBoxesHomePlacementMatchesTheEmptyCode(t *testing.T) {
home := haBox("i-0aaa", func(m map[string]any) { m["placement"] = "" })
named := haBox("i-0bbb", func(m map[string]any) { m["placement"] = "ie" })
snap := views(t, home, named)
- got := eligibleBoxes(snap, verifiedFor("i-0aaa", "i-0bbb"), "", "eu-west-2", "ie", testKeyFP, nil)
+ roster := rosterFor("i-0aaa", "i-0bbb")
+ for _, b := range []string{"i-0aaa", "i-0bbb"} {
+ rb := roster[b]
+ rb.Placement = "ie"
+ roster[b] = rb
+ }
+ got := eligibleBoxes(snap, roster, "", "eu-west-2", "ie", testKeyFP, nil)
if len(got) != 2 {
t.Fatalf("got %v, want both home boxes", ids(got))
}
}
func TestChooseBox(t *testing.T) {
- a, b, c := boxCandidate{BoxID: "i-0aaa"}, boxCandidate{BoxID: "i-0bbb"}, boxCandidate{BoxID: "i-0ccc"}
+ a, b := boxCandidate{BoxID: "i-0aaa"}, boxCandidate{BoxID: "i-0bbb"}
load := func(hosts int, apps ...string) boxLoad {
l := boxLoad{Hosts: hosts, Apps: map[string]int{}}
for _, x := range apps {
@@ -120,44 +129,82 @@ func TestChooseBox(t *testing.T) {
}
return l
}
- if _, ok := chooseBox(nil, nil, "bab/jam"); ok {
+ const host = "shop.bab.oncarlos.com"
+ if _, ok := chooseBox(host, nil, nil, "bab/jam"); ok {
t.Fatal("chose from nothing")
}
- got, _ := chooseBox([]boxCandidate{a, b}, map[string]boxLoad{"i-0aaa": load(5), "i-0bbb": load(2)}, "bab/jam")
+ got, _ := chooseBox(host, []boxCandidate{a, b}, map[string]boxLoad{"i-0aaa": load(5), "i-0bbb": load(2)}, "bab/jam")
if got.BoxID != "i-0bbb" {
t.Fatalf("fewest hosts: got %s", got.BoxID)
}
- got, _ = chooseBox([]boxCandidate{a, b}, map[string]boxLoad{"i-0aaa": load(5), "i-0bbb": load(2, "bab/jam")}, "bab/jam")
+ got, _ = chooseBox(host, []boxCandidate{a, b}, map[string]boxLoad{"i-0aaa": load(5), "i-0bbb": load(2, "bab/jam")}, "bab/jam")
if got.BoxID != "i-0aaa" {
t.Fatalf("spread beats load: got %s", got.BoxID)
}
- got, ok := chooseBox([]boxCandidate{a, b}, map[string]boxLoad{"i-0aaa": load(5, "bab/jam"), "i-0bbb": load(2, "bab/jam", "bab/jam")}, "bab/jam")
+ got, ok := chooseBox(host, []boxCandidate{a, b}, map[string]boxLoad{"i-0aaa": load(5, "bab/jam"), "i-0bbb": load(2, "bab/jam", "bab/jam")}, "bab/jam")
if !ok || got.BoxID != "i-0bbb" {
t.Fatalf("every box runs the app: fewest hosts overall, never a refusal; got %s ok=%v", got.BoxID, ok)
}
- got, _ = chooseBox([]boxCandidate{c, b, a}, nil, "bab/jam")
- if got.BoxID != "i-0aaa" {
- t.Fatalf("a tie breaks by box id: got %s", got.BoxID)
+}
+
+// H12, claim P5-D2: an equal load breaks by the greatest SHA-256 of
+// host + "\x00" + reference, so the choice differs host by host instead
+// of always landing on the lowest name, and does not depend on the order
+// the candidates arrive in.
+func TestChooseBoxBreaksATieByTheHostsHash(t *testing.T) {
+ cands := []boxCandidate{{BoxID: "annie-welding"}, {BoxID: "bert-sawing"}, {BoxID: "cleo-mixing"}}
+ want := func(host string) string {
+ best, bestSum := "", ""
+ for _, c := range cands {
+ sum := sha256.Sum256([]byte(host + "\x00" + c.BoxID))
+ if h := hex.EncodeToString(sum[:]); h > bestSum {
+ best, bestSum = c.BoxID, h
+ }
+ }
+ return best
+ }
+ seen := map[string]bool{}
+ for _, host := range []string{"a.bab.oncarlos.com", "b.bab.oncarlos.com", "c.bab.oncarlos.com", "d.bab.oncarlos.com", "e.bab.oncarlos.com", "f.bab.oncarlos.com"} {
+ forward, _ := chooseBox(host, cands, nil, "bab/jam")
+ backward, _ := chooseBox(host, []boxCandidate{cands[2], cands[1], cands[0]}, nil, "bab/jam")
+ if forward.BoxID != want(host) || backward.BoxID != want(host) {
+ t.Fatalf("%s: chose %s and %s, want %s", host, forward.BoxID, backward.BoxID, want(host))
+ }
+ seen[forward.BoxID] = true
+ }
+ if len(seen) < 2 {
+ t.Fatalf("six hosts all chose %v: the tie-break is not per host", seen)
}
}
-// A box whose SIGNED region is not the placement's AWS region is not in
-// that placement, even when its own file says it is and both regions have
-// certificates.
-func TestEligibleBoxesRefusesABoxSignedInAnotherRegion(t *testing.T) {
+// A box enrolled in another AWS region than the placement's is not in that
+// placement, even when its own file says it is.
+func TestEligibleBoxesRefusesABoxEnrolledInAnotherRegion(t *testing.T) {
snap := views(t, haBox("i-0aaa", nil), haBox("i-0bbb", nil))
- verified := verifiedFor("i-0aaa", "i-0bbb")
- verified["i-0bbb"] = boxid.Identity{AccountID: "1", Region: "us-east-1", InstanceID: "i-0bbb"}
- got := eligibleBoxes(snap, verified, "uk", "eu-west-2", "ie", testKeyFP, nil)
+ roster := rosterFor("i-0aaa", "i-0bbb")
+ roster["i-0bbb"] = rosterBox{Placement: "uk", Cloud: boxid.Cloud{Provider: "aws", Account: "1", Region: "us-east-1", InstanceID: "i-0bbb"}}
+ got := eligibleBoxes(snap, roster, "uk", "eu-west-2", "ie", testKeyFP, nil)
if len(got) != 1 || got[0].BoxID != "i-0aaa" {
t.Fatalf("got %v, want only i-0aaa", ids(got))
}
// And a file that advertises another AWS region than the one expected.
snap = views(t, haBox("i-0aaa", func(m map[string]any) { m["region"] = "us-east-1" }))
- if got := eligibleBoxes(snap, verifiedFor("i-0aaa"), "uk", "eu-west-2", "ie", testKeyFP, nil); len(got) != 0 {
+ if got := eligibleBoxes(snap, rosterFor("i-0aaa"), "uk", "eu-west-2", "ie", testKeyFP, nil); len(got) != 0 {
t.Fatalf("got %v, want none", ids(got))
}
- if got := eligibleBoxes(views(t, haBox("i-0aaa", nil)), verifiedFor("i-0aaa"), "uk", "", "ie", testKeyFP, nil); len(got) != 0 {
+ if got := eligibleBoxes(views(t, haBox("i-0aaa", nil)), rosterFor("i-0aaa"), "uk", "", "ie", testKeyFP, nil); len(got) != 0 {
t.Fatalf("got %v with no expected region", ids(got))
}
}
+
+// The roster's enrolled placement is the enrollment's word and must agree
+// with the file's; an incomplete cloud is no identity at all.
+func TestEligibleBoxesNeedsTheRosterToAgree(t *testing.T) {
+ snap := views(t, haBox("i-0aaa", nil), haBox("i-0bbb", nil))
+ roster := rosterFor("i-0aaa", "i-0bbb")
+ roster["i-0aaa"] = rosterBox{Placement: "us", Cloud: roster["i-0aaa"].Cloud}
+ roster["i-0bbb"] = rosterBox{Placement: "uk", Cloud: boxid.Cloud{Provider: "aws", Region: "eu-west-2", InstanceID: "i-0bbb"}}
+ if got := eligibleBoxes(snap, roster, "uk", "eu-west-2", "ie", testKeyFP, nil); len(got) != 0 {
+ t.Fatalf("got %v, want none", ids(got))
+ }
+}