amadan

carlos / platform Public

Merge pull request #335 from carlosframework/box-machine-join

The box tells us which machine it is
Paul Campbell pushed by paul@keymail.dev eb69ae64c16a43a6d4b1600422bfbc1d37a1f374 parent b0da3d8parent 09506c2
9 files changed, +400 −22
  • internal/awsinv/rest.go +27 −2
  • internal/awsinv/rest_test.go +21 −0
  • internal/console/archgraph.go +39 −8
  • internal/console/archjoin_test.go +89 −0
  • internal/host/agent.go +6 −5
  • internal/host/probes.go +32 −0
  • internal/status/instanceid.go +80 −0
  • internal/status/instanceid_test.go +76 −0
  • internal/status/snapshot.go +30 −7
diff --git a/internal/awsinv/rest.go b/internal/awsinv/rest.go
index 46073bfbf..5f33a1c4f 100644
--- a/internal/awsinv/rest.go
+++ b/internal/awsinv/rest.go
@@ -63,6 +63,31 @@ func s3Resources(ctx context.Context, api S3API) (out []infraview.Resource, part
return out, partial
}
+// unescapeDNSName turns Route53's wire form of a record name into the one
+// an operator recognises. The API returns non-ASCII and special characters
+// octal-escaped, so the wildcard every CARLOS deployment leans on comes back
+// as `\052.woodstar.app` — which is what the architecture page displayed
+// until someone looked at it. Only the printable-ASCII escapes are decoded;
+// anything else is left exactly as the API sent it, since a half-decoded
+// name would be worse than an escaped one.
+func unescapeDNSName(s string) string {
+ if !strings.Contains(s, `\`) {
+ return s
+ }
+ var b strings.Builder
+ for i := 0; i < len(s); i++ {
+ if s[i] == '\\' && i+3 < len(s) {
+ if n, err := strconv.ParseUint(s[i+1:i+4], 8, 8); err == nil && n >= 0x20 && n < 0x7f {
+ b.WriteByte(byte(n))
+ i += 3
+ continue
+ }
+ }
+ b.WriteByte(s[i])
+ }
+ return b.String()
+}
+
func route53Resources(ctx context.Context, api Route53API) (out []infraview.Resource, partial []string) {
zones, err := api.ListHostedZones(ctx, &route53.ListHostedZonesInput{})
if err != nil {
@@ -81,7 +106,7 @@ func route53Resources(ctx context.Context, api Route53API) (out []infraview.Reso
// dedup, keyed on the literal Resource.ID, find this zone without
// a second, service-specific seeding rule.
id := strings.TrimPrefix(aws.ToString(z.Id), "/hostedzone/")
- name := strings.TrimSuffix(aws.ToString(z.Name), ".")
+ name := unescapeDNSName(strings.TrimSuffix(aws.ToString(z.Name), "."))
zr := res(infraview.KindDNSZone, "route53:hosted-zone", id, name, "")
if z.Config != nil && z.Config.PrivateZone {
set(&zr, "private", "yes")
@@ -112,7 +137,7 @@ func route53Resources(ctx context.Context, api Route53API) (out []infraview.Reso
if !joinedRecordTypes[rtype] {
continue
}
- rname := strings.TrimSuffix(aws.ToString(rec.Name), ".")
+ rname := unescapeDNSName(strings.TrimSuffix(aws.ToString(rec.Name), "."))
for _, v := range rec.ResourceRecords {
rr := res(infraview.KindDNSRecord, "route53:record", rname+"/"+rtype+"/"+aws.ToString(v.Value), rname, "")
set(&rr, "zone", name)
diff --git a/internal/awsinv/rest_test.go b/internal/awsinv/rest_test.go
index 0e11c59b6..eadd4d0e9 100644
--- a/internal/awsinv/rest_test.go
+++ b/internal/awsinv/rest_test.go
@@ -472,3 +472,24 @@ func TestParseARN(t *testing.T) {
}
}
}
+
+// Route53 returns record and zone names octal-escaped, so the wildcard every
+// CARLOS deployment leans on arrives as `\052.woodstar.app`. It reached the
+// console's architecture page in that form and was read there by a person
+// before anyone noticed — decode it at the boundary that produces it.
+func TestDNSNamesAreUnescapedAtTheBoundary(t *testing.T) {
+ for _, c := range []struct{ in, want string }{
+ {`\052.woodstar.app`, "*.woodstar.app"},
+ {"plain.example.com", "plain.example.com"},
+ {`a\055b.example.com`, "a-b.example.com"},
+ // Not printable ASCII: left exactly as the API sent it, because a
+ // half-decoded name is worse than an escaped one.
+ {`\200.example.com`, `\200.example.com`},
+ // A trailing partial escape must not run off the end.
+ {`weird\05`, `weird\05`},
+ } {
+ if got := unescapeDNSName(c.in); got != c.want {
+ t.Errorf("unescapeDNSName(%q) = %q, want %q", c.in, got, c.want)
+ }
+ }
+}
diff --git a/internal/console/archgraph.go b/internal/console/archgraph.go
index 754921af5..fdcd2abf9 100644
--- a/internal/console/archgraph.go
+++ b/internal/console/archgraph.go
@@ -206,13 +206,37 @@ func pruneDanglingEdges(v *ArchView) {
// ---- the joins ---------------------------------------------------------
-// machineFor matches a machine's Name tag against a box label, case-
-// insensitively and exactly — the Name tag is set to the host's name and the
-// box label IS the box's hostname, so this join needs no convention beyond
-// the one the module already follows. Returns nil when nothing matches,
-// which is the BYO-box case and not an error.
-func machineFor(inv *infraview.Inventory, label string) *infraview.Resource {
- if inv == nil || label == "" {
+// machineFor finds the cloud machine a box runs on.
+//
+// The box's OWN report wins: status.BoxMetrics.InstanceID is the id it read
+// from the instance metadata service, and nothing else can be as sure. When
+// a box reports one, that id is the answer — a hit joins, and a MISS stays a
+// miss rather than falling through to a name match, because a box that has
+// told us which machine it is must not be quietly attached to a different
+// one. (A miss is real: an instance terminated between the box's last
+// snapshot and this inventory.)
+//
+// The Name-tag match is the fallback, and only for a box that reports no id
+// at all — an agent that predates the field, which during a roll is every
+// box until it restarts. It is a fallback rather than the rule because it
+// is a naming convention, and on the flagship that convention never held:
+// the instances are tagged carlos-flagship-ie while the boxes call
+// themselves annie-welding, so this function matched nothing at all and
+// every box on the page read as bring-your-own hardware.
+func machineFor(inv *infraview.Inventory, label string, reported string) *infraview.Resource {
+ if inv == nil {
+ return nil
+ }
+ if reported != "" {
+ for i := range inv.Resources {
+ r := &inv.Resources[i]
+ if r.Kind == infraview.KindMachine && r.ID == reported {
+ return r
+ }
+ }
+ return nil
+ }
+ if label == "" {
return nil
}
for i := range inv.Resources {
@@ -360,10 +384,17 @@ func buildBoxCards(in ArchInput) []BoxCard {
// Metrics is nil for a stale box by design (see HostRow) — an
// unreachable host's last happy numbers must never reach the page,
// and reading through the pointer is where that promise is kept.
+ // reportedID is the machine this box says it is. It comes from the
+ // same Metrics block as Release, so a STALE box reports neither —
+ // which is correct: a dark box's claim about which machine it was is
+ // exactly as old as its other readings, and the page falls back to
+ // the name match rather than trusting history.
+ var reportedID string
if host.Metrics != nil {
card.Release = host.Metrics.Release
+ reportedID = host.Metrics.InstanceID
}
- if m := machineFor(in.Inv, label); m != nil {
+ if m := machineFor(in.Inv, label, reportedID); m != nil {
card.CloudID = m.ID
card.InstanceType = m.Attr("type")
card.State = m.Attr("state")
diff --git a/internal/console/archjoin_test.go b/internal/console/archjoin_test.go
new file mode 100644
index 000000000..24d40be5d
--- /dev/null
+++ b/internal/console/archjoin_test.go
@@ -0,0 +1,89 @@
+package console
+
+import (
+ "testing"
+
+ "github.com/carlosframework/platform/internal/status"
+)
+
+// The join the flagship needs: the box reports the instance it runs on, and
+// that is matched against the machine's id. The Name tag is not consulted —
+// on the real fleet the instances are tagged carlos-flagship-ie while the
+// boxes call themselves annie-welding, and matching names found nothing at
+// all.
+func TestBoxJoinsMachineByReportedInstanceID(t *testing.T) {
+ in := archFixture()
+ for i := range in.Hosts {
+ if in.Hosts[i].Label == "carlos-eu-west-1" {
+ in.Hosts[i].Label = "annie-welding"
+ if in.Hosts[i].Metrics == nil {
+ in.Hosts[i].Metrics = &status.BoxMetrics{}
+ }
+ in.Hosts[i].Metrics.InstanceID = "i-123"
+ }
+ }
+ for i := range in.Fleet.Boxes {
+ if in.Fleet.Boxes[i].Label == "carlos-eu-west-1" {
+ in.Fleet.Boxes[i].Label = "annie-welding"
+ }
+ }
+ for i := range in.Inv.Resources {
+ if in.Inv.Resources[i].ID == "i-123" {
+ in.Inv.Resources[i].Name = "carlos-flagship-ie" // deliberately unlike the label
+ }
+ }
+
+ v := BuildArch(in)
+ var card BoxCard
+ for _, c := range v.Boxes {
+ if c.Label == "annie-welding" {
+ card = c
+ }
+ }
+ if card.CloudID != "i-123" {
+ t.Fatalf("CloudID = %q, want i-123 — the reported instance id must join regardless of naming", card.CloudID)
+ }
+ if card.InstanceType == "" {
+ t.Error("a joined box must carry the machine's type")
+ }
+ if hasNode(v, "machine:i-123") {
+ t.Error("a machine claimed by a box must not also be drawn as an unclaimed machine")
+ }
+}
+
+// The Name-tag match stays as a fallback for a box that reports no instance
+// id — an older agent that predates the field, mid-roll.
+func TestBoxStillJoinsByNameTagWhenNoInstanceIDReported(t *testing.T) {
+ v := BuildArch(archFixture())
+ var card BoxCard
+ for _, c := range v.Boxes {
+ if c.Label == "carlos-eu-west-1" {
+ card = c
+ }
+ }
+ if card.CloudID != "i-123" {
+ t.Fatalf("CloudID = %q, want the Name-tag fallback to still join", card.CloudID)
+ }
+}
+
+// A box reporting an instance id the estate does not contain is NOT then
+// matched by name to some other machine: the box has told us which machine
+// it is, and quietly joining it to a different one would be a worse answer
+// than admitting the miss.
+func TestReportedInstanceIDIsNotOverriddenByANameMatch(t *testing.T) {
+ in := archFixture()
+ for i := range in.Hosts {
+ if in.Hosts[i].Label == "carlos-eu-west-1" {
+ if in.Hosts[i].Metrics == nil {
+ in.Hosts[i].Metrics = &status.BoxMetrics{}
+ }
+ in.Hosts[i].Metrics.InstanceID = "i-terminated"
+ }
+ }
+ v := BuildArch(in)
+ for _, c := range v.Boxes {
+ if c.Label == "carlos-eu-west-1" && c.CloudID != "" {
+ t.Errorf("CloudID = %q, want empty: the box named a machine this estate does not have", c.CloudID)
+ }
+ }
+}
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 2c64e3e93..24d55b52f 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -2229,11 +2229,12 @@ func statusLoop(reg *registry.Registry) {
memTotal, memAvail := readMemBytes()
p := status.Probes{
- MachineID: readMachineID(),
- Region: boxRegion,
- MemFrac: readMemFrac(),
- DiskFrac: readDiskFrac(),
- Listening: wrapListeningProbe(unitBackedSockets(routes), listeningProbe),
+ MachineID: readMachineID(),
+ InstanceID: cachedInstanceID(),
+ Region: boxRegion,
+ MemFrac: readMemFrac(),
+ DiskFrac: readDiskFrac(),
+ Listening: wrapListeningProbe(unitBackedSockets(routes), listeningProbe),
// One systemctl call per 5-minute tick, asking the only question
// that catches an enabled-but-never-armed adoption timer — the
// flagship's silent no-adopt state, which nothing on any page
diff --git a/internal/host/probes.go b/internal/host/probes.go
index 3b3718eaa..481f376ee 100644
--- a/internal/host/probes.go
+++ b/internal/host/probes.go
@@ -17,6 +17,9 @@ import (
"os/exec"
"strconv"
"strings"
+ "sync"
+
+ "github.com/carlosframework/platform/internal/status"
"github.com/carlosframework/platform/internal/registry"
)
@@ -97,6 +100,35 @@ func parseIsActive(out string) *bool {
return nil
}
+// cachedInstanceID reads this box's cloud machine id once and remembers it
+// — the id cannot change under a running process, and the status loop asks
+// every five minutes.
+//
+// Only a SUCCESSFUL read is cached. An empty answer is retried next tick,
+// because the two reasons for one are not alike: off-cloud it stays empty
+// forever and costs a couple of unroutable milliseconds a tick, while on a
+// box whose metadata service was briefly unreachable at boot, caching the
+// empty would cost the console's box-to-machine join for the whole life of
+// the agent — cured only by a restart nobody would know to perform.
+//
+// This is the one probe in this file that reaches off the box, and it obeys
+// the same rule as the rest: it fails toward "", which CollectBox publishes
+// as absence, never a guess.
+var (
+ instanceIDMu sync.Mutex
+ instanceIDSeen string
+)
+
+func cachedInstanceID() string {
+ instanceIDMu.Lock()
+ defer instanceIDMu.Unlock()
+ if instanceIDSeen != "" {
+ return instanceIDSeen
+ }
+ instanceIDSeen = status.InstanceID()
+ return instanceIDSeen
+}
+
// readMachineID reads /etc/machine-id, trimmed. Unreadable (missing file,
// permission denied — plausible inside some containers) -> "", NOT a
// hash-of-hostname fallback. A hostname-derived id would look exactly like a
diff --git a/internal/status/instanceid.go b/internal/status/instanceid.go
new file mode 100644
index 000000000..ccbbd8532
--- /dev/null
+++ b/internal/status/instanceid.go
@@ -0,0 +1,80 @@
+package status
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+)
+
+// IMDSBase is the link-local instance metadata service every EC2 machine
+// answers on. It is a var only so the tests can point the reader at an
+// httptest server; nothing else reassigns it.
+var IMDSBase = "http://169.254.169.254"
+
+// imdsTimeout bounds the whole two-request exchange. It is deliberately
+// short: off-cloud the address is unroutable and the only honest answer is
+// "no instance id", which must not cost the status loop its tick. The
+// link-local service answers in single-digit milliseconds when it answers
+// at all.
+const imdsTimeout = 2 * time.Second
+
+// InstanceID is the id of the EC2 instance this process runs on, or "" for
+// anything that is not one — a Pi on a desk, a laptop, a container, another
+// cloud.
+//
+// It exists because the console's architecture page has to answer "which
+// machine is this box?", and every other way of asking is a naming
+// convention that can drift: the flagship's instances are tagged
+// carlos-flagship-ie while its boxes call themselves annie-welding, so a
+// Name-tag match found nothing at all. The box is the only thing that knows
+// which machine it is without being told, and this is it asking.
+//
+// The empty string is a real answer, not a failure: it is what tells the
+// console this is somebody else's hardware rather than a machine it looked
+// for and missed.
+func InstanceID() string { return instanceIDFrom(IMDSBase, imdsTimeout) }
+
+// instanceIDFrom speaks IMDSv2 — a PUT for a short-lived token, then a GET
+// carrying it. IMDSv1 (a bare GET) is not attempted: every box this
+// platform provisions has v2 enforced, and falling back would only make an
+// unreachable service take twice as long to say nothing.
+func instanceIDFrom(base string, timeout time.Duration) string {
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+ defer cancel()
+ client := &http.Client{Timeout: timeout}
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodPut, base+"/latest/api/token", nil)
+ if err != nil {
+ return ""
+ }
+ req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "60")
+ resp, err := client.Do(req)
+ if err != nil {
+ return ""
+ }
+ token, err := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ resp.Body.Close()
+ if err != nil || resp.StatusCode != http.StatusOK || len(token) == 0 {
+ return ""
+ }
+
+ req, err = http.NewRequestWithContext(ctx, http.MethodGet, base+"/latest/meta-data/instance-id", nil)
+ if err != nil {
+ return ""
+ }
+ req.Header.Set("X-aws-ec2-metadata-token", string(token))
+ resp, err = client.Do(req)
+ if err != nil {
+ return ""
+ }
+ body, err := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ resp.Body.Close()
+ if err != nil || resp.StatusCode != http.StatusOK {
+ return ""
+ }
+ // Trimmed because a stray newline would make the console's join miss by
+ // one byte, which is the most annoying possible way for this to fail.
+ return strings.TrimSpace(string(body))
+}
diff --git a/internal/status/instanceid_test.go b/internal/status/instanceid_test.go
new file mode 100644
index 000000000..012a5a9dd
--- /dev/null
+++ b/internal/status/instanceid_test.go
@@ -0,0 +1,76 @@
+package status
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+)
+
+// A box that knows nothing but its own instance id still publishes a
+// Metrics block: the id is the join the console's architecture page hangs
+// the whole cloud half on, and the attach rule must not drop it for want of
+// a memory reading.
+func TestCollectBoxPublishesInstanceIDAlone(t *testing.T) {
+ box := CollectBox(nil, Probes{InstanceID: "i-0c8e41d9"}, time.Now())
+ if box.Metrics == nil {
+ t.Fatal("a box that knows its instance id must publish a Metrics block")
+ }
+ if box.Metrics.InstanceID != "i-0c8e41d9" {
+ t.Errorf("InstanceID = %q, want i-0c8e41d9", box.Metrics.InstanceID)
+ }
+}
+
+// Off-cloud there is no instance id, and absence is the honest answer: it
+// is what tells the console this is somebody else's hardware rather than a
+// machine it failed to find.
+func TestCollectBoxOmitsAnAbsentInstanceID(t *testing.T) {
+ box := CollectBox(nil, Probes{}, time.Now())
+ if box.Metrics != nil && box.Metrics.InstanceID != "" {
+ t.Errorf("InstanceID = %q, want empty", box.Metrics.InstanceID)
+ }
+}
+
+// InstanceID reads IMDSv2: a PUT for a token, then a GET carrying it. A
+// server that refuses the token (IMDSv1-only, or the uid-gate) yields "",
+// never a partial or a panic.
+func TestInstanceIDReadsIMDSv2(t *testing.T) {
+ var sawToken bool
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodPut && r.URL.Path == "/latest/api/token":
+ if r.Header.Get("X-aws-ec2-metadata-token-ttl-seconds") == "" {
+ t.Error("token request must carry a TTL header")
+ }
+ w.Write([]byte("tok123"))
+ case r.URL.Path == "/latest/meta-data/instance-id":
+ if r.Header.Get("X-aws-ec2-metadata-token") == "tok123" {
+ sawToken = true
+ }
+ w.Write([]byte("i-0abcdef"))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer srv.Close()
+
+ if got := instanceIDFrom(srv.URL, time.Second); got != "i-0abcdef" {
+ t.Errorf("instanceIDFrom = %q, want i-0abcdef", got)
+ }
+ if !sawToken {
+ t.Error("the metadata GET must present the IMDSv2 token")
+ }
+}
+
+func TestInstanceIDIsEmptyWhenIMDSIsUnreachable(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "denied", http.StatusForbidden)
+ }))
+ defer srv.Close()
+ if got := instanceIDFrom(srv.URL, time.Second); got != "" {
+ t.Errorf("instanceIDFrom = %q, want empty when IMDS refuses", got)
+ }
+ if got := instanceIDFrom("http://127.0.0.1:1", 200*time.Millisecond); got != "" {
+ t.Errorf("instanceIDFrom = %q, want empty when nothing answers", got)
+ }
+}
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go
index a78cf5873..a69318d6a 100644
--- a/internal/status/snapshot.go
+++ b/internal/status/snapshot.go
@@ -39,12 +39,16 @@ import (
// Probes carries everything CollectBox may not read from the registry —
// each nil means "unknown", and unknown publishes as null, never a guess.
type Probes struct {
- MachineID string // hashed into the label; never serialized
- Region string // operator-set, coarse; "" omits the field
- MemFrac *float64 // 0..1 memory utilization
- DiskFrac *float64 // 0..1 proximity to the disk floor
- Listening func(sock string) *bool // kernel socket table read; nil fn = unknown
- LagSecs *int64 // seconds since litestream last progressed
+ MachineID string // hashed into the label; never serialized
+ // InstanceID is the cloud machine this box runs on (status.InstanceID),
+ // or "" for hardware no cloud owns. It is the console architecture
+ // page's join key — see BoxMetrics.InstanceID.
+ InstanceID string
+ Region string // operator-set, coarse; "" omits the field
+ MemFrac *float64 // 0..1 memory utilization
+ DiskFrac *float64 // 0..1 proximity to the disk floor
+ Listening func(sock string) *bool // kernel socket table read; nil fn = unknown
+ LagSecs *int64 // seconds since litestream last progressed
// AdoptRunning is whether release adoption is actually armed on this box
// — the carlos-adopt.timer being ACTIVE, not merely enabled. nil is
// "never asked / couldn't tell", and like every other nil here it must
@@ -152,7 +156,24 @@ type BoxMetrics struct {
Load1 *float64 `json:"load1,omitempty"`
DiskFreeBytes *int64 `json:"disk_free_bytes,omitempty"`
Release string `json:"release,omitempty"` // running binary's version.Build
- Tunnel string `json:"tunnel,omitempty"` // ""|"connected"|"reconnecting"
+ // InstanceID is the cloud machine this box IS — read from the instance
+ // metadata service by the box itself (status.InstanceID). "" means no
+ // cloud machine owns it: a Pi, a laptop, someone else's hardware.
+ //
+ // It lives here, in the private per-box block, for the reason every
+ // hostname-bearing field here does: status/boxes/<label>.json is
+ // bucket-private and BuildFleet never copies Metrics into the public
+ // feed, so an instance id cannot reach fleet.json.
+ //
+ // The console's architecture page joins boxes to machines on this and
+ // nothing else matters as much: the first version matched an instance's
+ // Name tag against the box label, and on the flagship those namespaces
+ // are disjoint (carlos-flagship-ie versus annie-welding), so the join
+ // found nothing at all and every box read as bring-your-own. A box
+ // asking the metadata service which machine it is cannot drift from a
+ // naming convention, because it is not using one.
+ InstanceID string `json:"instance_id,omitempty"`
+ Tunnel string `json:"tunnel,omitempty"` // ""|"connected"|"reconnecting"
// RouteVersions is host -> registry.Route.Version for every non-platform
// route this box carries with a Version the adopt pass has stamped
// (issue #78's staged_version chip) — a hostname-keyed map, which is
@@ -520,11 +541,13 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {
// alias) has nothing to say, not something empty to say.
if p.MemTotalBytes != nil || p.MemAvailBytes != nil || p.Load1 != nil ||
p.DiskFreeBytes != nil || p.Release != "" || p.Tunnel != "" ||
+ p.InstanceID != "" ||
len(routeVersions) > 0 || len(aliasReadings) > 0 || len(p.Upstreams) > 0 {
box.Metrics = &BoxMetrics{
MemTotalBytes: p.MemTotalBytes, MemAvailBytes: p.MemAvailBytes,
Load1: p.Load1, DiskFreeBytes: p.DiskFreeBytes,
Release: p.Release, Tunnel: p.Tunnel,
+ InstanceID: p.InstanceID,
RouteVersions: routeVersions,
Aliases: aliasReadings,
// Carried through as the prober handed it over: proberSet.State