amadan

carlos / platform Public

steering: route53 converger — latency records + health checks, bounded to managed set

Converge(ctx, client, zoneID, want) makes a hosted zone's carlos-managed
latency A records match the desired set from Task 4, and mints/GCs the
Route53 health checks those records bind to. No caller yet — Task 6 wires
it into the host housekeeper, so this is single-box inert.

Deletion safety is the design. Two boundaries do all the work:

  - isManaged: a record is deletable only if it is an A record AND carries
    a SetIdentifier (a record without one is static/operator territory)
    AND is a latency record AND is not an alias AND is bound to a health
    check this deployment owns. The last gate makes "can it delete a
    record it didn't create?" a structural no, at the cost of leaving an
    orphan (loudly logged) if a check goes missing.
  - ownedHealthChecks: ownership is the carlos-steering=<zoneID> TAG, read
    per check. A shape prefilter narrows the candidate set first, which can
    only ever shrink what is deletable. An untagged check, or one tagged
    for another zone, is invisible to the GC.

Also: an empty desired set is refused, not obeyed — desiredState returns
nothing when zero edges are armed, and a degraded read must not become a
wholesale teardown of production DNS. A create-then-tag whose tagging fails
deletes the check it just made, so a failure cannot leak one per pass.

Records are upserted only when they differ, so a steady-state pass makes
zero write calls; changes are chunked (upserts before deletes) and a failed
batch fails the pass, leaving the GC unrun for the next tick to redo.

go.mod is unchanged: aws-sdk-go-v2/service/route53 was already a direct
dependency via internal/edge/route53dns.go.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell pushed by paul@keymail.dev 7ca22b106d89fdbd722f520a70383955bd6d467f parent ac2e728
2 files changed, +1531 −0
  • internal/steering/route53.go +713 −0
  • internal/steering/route53_test.go +818 −0
diff --git a/internal/steering/route53.go b/internal/steering/route53.go
new file mode 100644
index 000000000..00adefda6
--- /dev/null
+++ b/internal/steering/route53.go
@@ -0,0 +1,713 @@
+package steering
+
+// route53.go — the converger. This is the ONLY file in the tree that holds
+// write credentials to a production DNS zone, so its whole design is about
+// what it REFUSES to touch. Read the managed-set rule (isManaged) and the
+// health-check ownership rule (ownedHealthChecks) before changing anything
+// here: everything else is bookkeeping around those two boundaries.
+//
+// SINGLE-BOX INERT as of this task: nothing calls Converge. Task 6 wires it
+// into the host housekeeper, where CARLOS_STEERING_ZONE_ID unset (every box
+// today) keeps the duty dormant — zero API calls, zero writes.
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+ "regexp"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awsconfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/service/route53"
+ r53types "github.com/aws/aws-sdk-go-v2/service/route53/types"
+)
+
+const (
+ // steeringZoneEnv names the deployment's steering hosted zone. Unset =
+ // dormant, which is every box today.
+ steeringZoneEnv = "CARLOS_STEERING_ZONE_ID"
+
+ // steeringTagKey / its value (the zone id) is the ONLY thing that makes
+ // a Route53 health check eligible for deletion by this code. An untagged
+ // check, or one tagged for another deployment's zone, is invisible to
+ // the GC — see ownedHealthChecks.
+ steeringTagKey = "carlos-steering"
+
+ // recordTTL is short on purpose: the health check withdraws a dead
+ // edge's answer in ~90s, and a long TTL would keep resolvers handing it
+ // out for far longer than that.
+ recordTTL = 60
+
+ // health check shape. HTTPS on 443 with SNI: the name in
+ // FullyQualifiedDomainName is what the edge sees in the handshake, so it
+ // must be a name the edge holds a certificate for.
+ healthCheckPort = 443
+ healthCheckPath = "/"
+ healthCheckInterval = 30 // seconds
+ healthCheckFailureThresh = 3 // 3 x 30s ~= the spec's 90s withdrawal
+ healthCheckTagBatch = 10 // ListTagsForResources' own cap
+ healthCheckCallerRefPrefix = "carlos-steering-"
+
+ // changeBatchLimit chunks ChangeResourceRecordSets well under Route53's
+ // own 1000-change cap. Smaller batches mean a failure retries less work
+ // and names a smaller blast radius in the log.
+ changeBatchLimit = 100
+)
+
+// steeringZonePattern is the region-host tofu module's own zone-id shape
+// (infra/modules/region-host/variables.tf: `^Z[A-Z0-9]+$`) — a BARE id, never
+// the "/hostedzone/Z..." ARN-ish form the API returns in some places. A typo
+// that slips this gate would be a write against some other zone, so the gate
+// runs before any client call, not after.
+var steeringZonePattern = regexp.MustCompile(`^Z[A-Z0-9]+$`)
+
+// route53API is the slice of the Route53 client this converger uses — an
+// interface so tests substitute an in-memory fake and never reach AWS
+// (route53_test.go). Same seam as internal/edge/route53dns.go's own
+// route53API.
+type route53API interface {
+ ListResourceRecordSets(ctx context.Context, in *route53.ListResourceRecordSetsInput, optFns ...func(*route53.Options)) (*route53.ListResourceRecordSetsOutput, error)
+ ChangeResourceRecordSets(ctx context.Context, in *route53.ChangeResourceRecordSetsInput, optFns ...func(*route53.Options)) (*route53.ChangeResourceRecordSetsOutput, error)
+ ListHealthChecks(ctx context.Context, in *route53.ListHealthChecksInput, optFns ...func(*route53.Options)) (*route53.ListHealthChecksOutput, error)
+ ListTagsForResources(ctx context.Context, in *route53.ListTagsForResourcesInput, optFns ...func(*route53.Options)) (*route53.ListTagsForResourcesOutput, error)
+ CreateHealthCheck(ctx context.Context, in *route53.CreateHealthCheckInput, optFns ...func(*route53.Options)) (*route53.CreateHealthCheckOutput, error)
+ DeleteHealthCheck(ctx context.Context, in *route53.DeleteHealthCheckInput, optFns ...func(*route53.Options)) (*route53.DeleteHealthCheckOutput, error)
+ ChangeTagsForResource(ctx context.Context, in *route53.ChangeTagsForResourceInput, optFns ...func(*route53.Options)) (*route53.ChangeTagsForResourceOutput, error)
+}
+
+// SteeringZoneID returns the deployment's validated steering hosted-zone id,
+// or "" when the deployment is dormant (env unset, blank, or malformed —
+// malformed is logged, because a typo'd zone id is a configuration error
+// somebody needs to see, not a quiet no-op).
+func SteeringZoneID() string {
+ raw := strings.TrimSpace(os.Getenv(steeringZoneEnv))
+ if raw == "" {
+ return ""
+ }
+ if !steeringZonePattern.MatchString(raw) {
+ log.Printf("steering: %s=%q is not a bare hosted-zone id (want e.g. Z0ABCDEF1234) — staying dormant", steeringZoneEnv, raw)
+ return ""
+ }
+ return raw
+}
+
+// NewClient builds a Route53 client from the default AWS credential chain —
+// the box instance profile in production, exactly like
+// internal/edge/route53dns.go's newRoute53DNS. Route53 is a global service
+// but the SDK still needs a signing region, so default to us-east-1 when the
+// environment names none; an explicit AWS_REGION still wins.
+func NewClient(ctx context.Context) (*route53.Client, error) {
+ var opts []func(*awsconfig.LoadOptions) error
+ if strings.TrimSpace(os.Getenv("AWS_REGION")) == "" && strings.TrimSpace(os.Getenv("AWS_DEFAULT_REGION")) == "" {
+ opts = append(opts, awsconfig.WithRegion("us-east-1"))
+ }
+ cfg, err := awsconfig.LoadDefaultConfig(ctx, opts...)
+ if err != nil {
+ return nil, fmt.Errorf("steering: load aws config: %w", err)
+ }
+ return route53.NewFromConfig(cfg), nil
+}
+
+// ownedCheck is one Route53 health check this deployment owns: tagged
+// carlos-steering=<zoneID>, and therefore the only kind this code will ever
+// delete.
+type ownedCheck struct {
+ id string
+ eip string
+}
+
+// Converge makes the zone's carlos-managed latency records match want.
+//
+// The order is deliberate and is a safety property, not an implementation
+// detail:
+//
+// 1. read ownership (which health checks are OURS) — a failure here aborts
+// before anything is written, and an empty result means the managed
+// record set is empty, which means NOTHING is deletable;
+// 2. read the zone;
+// 3. CREATE any missing health check — capacity before records need it;
+// 4. UPSERT wanted records, DELETE managed-but-unwanted ones;
+// 5. GC health checks nothing references any more — after the records that
+// referenced them are gone, never before.
+//
+// A failure at any step returns loudly and leaves the rest of the pass
+// undone; the housekeeper's next tick re-runs the whole thing from a fresh
+// read. Every step is individually idempotent, so a half-applied pass is a
+// state this function converges from, not a state it gets stuck in.
+func Converge(ctx context.Context, client route53API, zoneID string, want []desiredRecord) error {
+ if !steeringZonePattern.MatchString(strings.TrimSpace(zoneID)) {
+ return fmt.Errorf("steering: refusing to converge: %q is not a bare hosted-zone id", zoneID)
+ }
+ zoneID = strings.TrimSpace(zoneID)
+
+ // An EMPTY desired set is never an instruction to tear down production
+ // DNS. desiredState only returns nothing when zero edges are armed (or
+ // there are no steered hosts AND no apps domain), and the overwhelmingly
+ // likelier cause of that on a live deployment is a degraded read — a
+ // control-bucket blip, a listing that came back short — not an operator
+ // deciding every edge should stop answering. Deleting the whole managed
+ // set on that signal would turn a transient read failure into a global
+ // outage. So: refuse, loudly, and let the next tick decide again.
+ // A genuine full teardown is an operator action.
+ if len(want) == 0 {
+ log.Printf("steering: empty desired set for zone %s — refusing to remove any record (a wholesale teardown is an operator action, not a converge outcome)", zoneID)
+ return nil
+ }
+
+ owned, err := ownedHealthChecks(ctx, client, zoneID)
+ if err != nil {
+ return err
+ }
+ ownedIDs := make(map[string]bool, len(owned))
+ checkByEIP := make(map[string]string, len(owned))
+ for _, c := range owned {
+ ownedIDs[c.id] = true
+ // Deterministic winner when a leak has left two checks on one EIP:
+ // lowest id. The loser is unreferenced after this pass and the GC
+ // below collects it — the leak self-heals.
+ if have, ok := checkByEIP[c.eip]; !ok || c.id < have {
+ checkByEIP[c.eip] = c.id
+ }
+ }
+
+ existing, err := listRecords(ctx, client, zoneID)
+ if err != nil {
+ return err
+ }
+
+ // 3. health checks for every wanted EIP.
+ sni := steeringHostname(want)
+ for _, eip := range distinctEIPs(want) {
+ if _, ok := checkByEIP[eip]; ok {
+ continue
+ }
+ fqdnFor := sni
+ if fqdnFor == "" {
+ fqdnFor = fallbackHostname(want, eip)
+ }
+ id, err := createHealthCheck(ctx, client, zoneID, eip, fqdnFor)
+ if err != nil {
+ return err
+ }
+ checkByEIP[eip] = id
+ ownedIDs[id] = true
+ owned = append(owned, ownedCheck{id: id, eip: eip})
+ }
+
+ // 4. records.
+ changes, err := recordChanges(want, existing, checkByEIP, ownedIDs)
+ if err != nil {
+ return err
+ }
+ if err := applyChanges(ctx, client, zoneID, changes); err != nil {
+ return err
+ }
+
+ // 5. GC. Runs only once the record batches have landed.
+ return gcHealthChecks(ctx, client, owned, want, existing, changes)
+}
+
+// ---------------------------------------------------------------------------
+// ownership
+// ---------------------------------------------------------------------------
+
+// ownedHealthChecks returns the account's health checks that THIS deployment
+// owns, i.e. that carry carlos-steering=<zoneID>.
+//
+// Two narrowings, both in the safe direction:
+//
+// - the shape prefilter (HTTPS on 443 with an IPAddress, and not owned by
+// another AWS service) only ever SHRINKS the set of checks that can be
+// deleted. Every check this code creates has exactly that shape, so a
+// tagged check of some other shape is one we did not create — ignoring it
+// costs a stranded check; deleting it could cost someone else's
+// monitoring.
+// - the tag is then checked for real, per candidate. Shape alone never
+// confers ownership: the tag is the authority.
+//
+// A listing or tag call that errors aborts the whole pass. That matters: if
+// this function returned a partial set, checks it failed to see would look
+// unowned (so records bound to them would look unmanaged — harmless) but,
+// worse, their EIPs would look uncovered and a duplicate check would be
+// created. Failing loudly is the only correct response.
+func ownedHealthChecks(ctx context.Context, client route53API, zoneID string) ([]ownedCheck, error) {
+ type candidate struct{ id, eip string }
+ var candidates []candidate
+
+ var marker *string
+ for {
+ out, err := client.ListHealthChecks(ctx, &route53.ListHealthChecksInput{Marker: marker})
+ if err != nil {
+ return nil, fmt.Errorf("steering: list health checks: %w", err)
+ }
+ for _, hc := range out.HealthChecks {
+ cfg := hc.HealthCheckConfig
+ if cfg == nil || hc.LinkedService != nil {
+ continue
+ }
+ if cfg.Type != r53types.HealthCheckTypeHttps || aws.ToInt32(cfg.Port) != healthCheckPort {
+ continue
+ }
+ eip := aws.ToString(cfg.IPAddress)
+ if eip == "" {
+ continue
+ }
+ candidates = append(candidates, candidate{id: aws.ToString(hc.Id), eip: eip})
+ }
+ if !out.IsTruncated || aws.ToString(out.NextMarker) == "" {
+ break
+ }
+ marker = out.NextMarker
+ }
+ if len(candidates) == 0 {
+ return nil, nil
+ }
+ sort.Slice(candidates, func(i, j int) bool { return candidates[i].id < candidates[j].id })
+
+ eipByID := make(map[string]string, len(candidates))
+ ids := make([]string, 0, len(candidates))
+ for _, c := range candidates {
+ eipByID[c.id] = c.eip
+ ids = append(ids, c.id)
+ }
+
+ var owned []ownedCheck
+ for start := 0; start < len(ids); start += healthCheckTagBatch {
+ end := start + healthCheckTagBatch
+ if end > len(ids) {
+ end = len(ids)
+ }
+ out, err := client.ListTagsForResources(ctx, &route53.ListTagsForResourcesInput{
+ ResourceType: r53types.TagResourceTypeHealthcheck,
+ ResourceIds: ids[start:end],
+ })
+ if err != nil {
+ return nil, fmt.Errorf("steering: list health-check tags: %w", err)
+ }
+ for _, set := range out.ResourceTagSets {
+ id := aws.ToString(set.ResourceId)
+ for _, tg := range set.Tags {
+ if aws.ToString(tg.Key) == steeringTagKey && aws.ToString(tg.Value) == zoneID {
+ owned = append(owned, ownedCheck{id: id, eip: eipByID[id]})
+ break
+ }
+ }
+ }
+ }
+ sort.Slice(owned, func(i, j int) bool { return owned[i].id < owned[j].id })
+ return owned, nil
+}
+
+// createHealthCheck mints one check for eip and tags it for this zone.
+//
+// Create-then-tag is two calls and Route53 offers no atomic form. An
+// UNTAGGED check is the worst outcome available: this code would never reuse
+// it (untagged is not ours) and never GC it (the GC is tag-scoped), so every
+// subsequent pass would mint another one — an unbounded leak at the
+// housekeeper's cadence. So a tagging failure best-effort deletes the check
+// it just made before failing the pass. If that delete ALSO fails, the check
+// is named in the log so an operator can find it.
+func createHealthCheck(ctx context.Context, client route53API, zoneID, eip, fqdn string) (string, error) {
+ // CallerReference must be unique per creation: Route53 remembers it for
+ // deleted checks for a while, so a name derived only from the EIP would
+ // make a delete-then-recreate (edge disarms, then rearms) fail with
+ // HealthCheckAlreadyExists. The timestamp keeps it under the 64-char cap
+ // while staying unique. Ownership is NEVER read from this string — the
+ // tag is the authority — so its only job is uniqueness plus a human hint
+ // in the console.
+ ref := fmt.Sprintf("%s%s-%d", healthCheckCallerRefPrefix, eip, time.Now().UnixNano())
+ out, err := client.CreateHealthCheck(ctx, &route53.CreateHealthCheckInput{
+ CallerReference: aws.String(ref),
+ HealthCheckConfig: &r53types.HealthCheckConfig{
+ Type: r53types.HealthCheckTypeHttps,
+ Port: aws.Int32(healthCheckPort),
+ IPAddress: aws.String(eip),
+ FullyQualifiedDomainName: aws.String(fqdn),
+ ResourcePath: aws.String(healthCheckPath),
+ EnableSNI: aws.Bool(true),
+ RequestInterval: aws.Int32(healthCheckInterval),
+ FailureThreshold: aws.Int32(healthCheckFailureThresh),
+ MeasureLatency: aws.Bool(false),
+ },
+ })
+ if err != nil {
+ return "", fmt.Errorf("steering: create health check for %s: %w", eip, err)
+ }
+ id := ""
+ if out.HealthCheck != nil {
+ id = aws.ToString(out.HealthCheck.Id)
+ }
+ if id == "" {
+ return "", fmt.Errorf("steering: create health check for %s: no id returned", eip)
+ }
+
+ if _, err := client.ChangeTagsForResource(ctx, &route53.ChangeTagsForResourceInput{
+ ResourceType: r53types.TagResourceTypeHealthcheck,
+ ResourceId: aws.String(id),
+ AddTags: []r53types.Tag{{Key: aws.String(steeringTagKey), Value: aws.String(zoneID)}},
+ }); err != nil {
+ if _, delErr := client.DeleteHealthCheck(ctx, &route53.DeleteHealthCheckInput{HealthCheckId: aws.String(id)}); delErr != nil {
+ log.Printf("steering: LEAKED health check %s for %s: tagging failed (%v) and so did the cleanup delete (%v) — delete it by hand", id, eip, err, delErr)
+ }
+ return "", fmt.Errorf("steering: tag health check %s for %s: %w", id, eip, err)
+ }
+ log.Printf("steering: created health check %s for %s (HTTPS:%d, SNI %s), tagged %s=%s", id, eip, healthCheckPort, fqdn, steeringTagKey, zoneID)
+ return id, nil
+}
+
+// gcHealthChecks deletes owned checks that nothing references any more.
+//
+// Two independent gates, both of which must pass:
+//
+// - the check is OURS (it is in owned, which is tag-derived — see
+// ownedHealthChecks). This is the gate that makes it structurally
+// impossible to delete a check belonging to another deployment, another
+// tool, or a person.
+// - no record SURVIVING this pass references it: not one of the records we
+// just wrote, and not one of the zone's records we left alone (which
+// includes every static and foreign record). Route53 would refuse a
+// delete of an in-use check anyway, but failing the whole pass on that
+// is worse than skipping the one check and saying so.
+//
+// A delete that fails is logged, not fatal: the records are already correct,
+// and next tick will try again.
+func gcHealthChecks(ctx context.Context, client route53API, owned []ownedCheck, want []desiredRecord, existing []r53types.ResourceRecordSet, applied []r53types.Change) error {
+ if len(owned) == 0 {
+ return nil
+ }
+
+ wantedEIP := make(map[string]bool, len(want))
+ for _, rec := range want {
+ wantedEIP[rec.EIP] = true
+ }
+
+ // Every check id still referenced once this pass has landed.
+ deleted := make(map[string]bool)
+ for _, c := range applied {
+ if c.Action == r53types.ChangeActionDelete && c.ResourceRecordSet != nil {
+ deleted[recordKey(*c.ResourceRecordSet)] = true
+ }
+ }
+ referenced := make(map[string]bool)
+ for _, rr := range existing {
+ if deleted[recordKey(rr)] {
+ continue
+ }
+ if id := aws.ToString(rr.HealthCheckId); id != "" {
+ referenced[id] = true
+ }
+ }
+ for _, c := range applied {
+ if c.Action == r53types.ChangeActionUpsert && c.ResourceRecordSet != nil {
+ if id := aws.ToString(c.ResourceRecordSet.HealthCheckId); id != "" {
+ referenced[id] = true
+ }
+ }
+ }
+
+ for _, c := range owned {
+ if wantedEIP[c.eip] && referenced[c.id] {
+ continue
+ }
+ if referenced[c.id] {
+ log.Printf("steering: health check %s (%s) is no longer wanted but a surviving record still references it — leaving it in place", c.id, c.eip)
+ continue
+ }
+ if _, err := client.DeleteHealthCheck(ctx, &route53.DeleteHealthCheckInput{HealthCheckId: aws.String(c.id)}); err != nil {
+ log.Printf("steering: could not delete unreferenced health check %s (%s): %v — retrying next pass", c.id, c.eip, err)
+ continue
+ }
+ log.Printf("steering: deleted unreferenced health check %s (%s)", c.id, c.eip)
+ }
+ return nil
+}
+
+// ---------------------------------------------------------------------------
+// records
+// ---------------------------------------------------------------------------
+
+// listRecords pages the whole zone. Every page matters: a converger that
+// stopped early would see records it manages as absent and rewrite them, and
+// would see managed-but-unwanted ones as already gone.
+func listRecords(ctx context.Context, client route53API, zoneID string) ([]r53types.ResourceRecordSet, error) {
+ var out []r53types.ResourceRecordSet
+ in := &route53.ListResourceRecordSetsInput{HostedZoneId: aws.String(zoneID)}
+ for {
+ page, err := client.ListResourceRecordSets(ctx, in)
+ if err != nil {
+ return nil, fmt.Errorf("steering: list record sets in %s: %w", zoneID, err)
+ }
+ out = append(out, page.ResourceRecordSets...)
+ if !page.IsTruncated || aws.ToString(page.NextRecordName) == "" {
+ return out, nil
+ }
+ in = &route53.ListResourceRecordSetsInput{
+ HostedZoneId: aws.String(zoneID),
+ StartRecordName: page.NextRecordName,
+ StartRecordType: page.NextRecordType,
+ StartRecordIdentifier: page.NextRecordIdentifier,
+ }
+ }
+}
+
+// recordKey is a record set's identity: Route53 keys a resource record set on
+// (name, type, set identifier), and so does this converger. Names are
+// lowercased because DNS is case-insensitive and desiredRecord.Name is
+// already lowercase by contract (steering.go's fqdn) — the listing side is
+// the only place a stray case could enter.
+func recordKey(rr r53types.ResourceRecordSet) string {
+ return strings.ToLower(aws.ToString(rr.Name)) + "|" + string(rr.Type) + "|" + aws.ToString(rr.SetIdentifier)
+}
+
+func wantKey(rec desiredRecord) string {
+ return rec.Name + "|" + string(r53types.RRTypeA) + "|" + rec.Box
+}
+
+// isManaged is THE deletion boundary. A record in the steering zone is
+// carlos-managed — and therefore something this code may delete — only if ALL
+// of the following hold:
+//
+// - it is an A record. Nothing else is ever written here.
+// - it carries a SetIdentifier. A record WITHOUT one is a plain,
+// single-answer record: the zone's apex, its www, its MX/NS/SOA, an
+// operator's hand-made host. Those are STATIC / OPERATOR TERRITORY and
+// are never listed as deletable, full stop. This is the rule the brief
+// names and the one that must never be relaxed.
+// - it is a LATENCY record (Region set). A weighted, failover, geo or
+// multivalue record also carries a SetIdentifier and is somebody else's
+// routing policy.
+// - it is not an alias. This converger writes plain A records to EIPs;
+// an alias record under our key was made by something else.
+// - it is bound to a health check WE own (tag-derived, ownedHealthChecks).
+// This is the strongest of the five: every record this code writes is
+// bound to one of our tagged checks, so the binding is a creation
+// signature. It is what makes the answer to "can Converge delete a
+// record it did not create?" a structural no rather than a naming
+// convention.
+//
+// The last gate has a cost, taken deliberately: a managed record whose health
+// check was deleted out from under us becomes invisible to this code and will
+// linger. It is logged by name every pass (see recordChanges) so it is loud
+// rather than silent, and a stale record is a far cheaper failure than a
+// wrongly-deleted one.
+func isManaged(rr r53types.ResourceRecordSet, ownedIDs map[string]bool) bool {
+ if rr.Type != r53types.RRTypeA {
+ return false
+ }
+ if aws.ToString(rr.SetIdentifier) == "" {
+ return false
+ }
+ if rr.Region == "" {
+ return false
+ }
+ if rr.AliasTarget != nil {
+ return false
+ }
+ return ownedIDs[aws.ToString(rr.HealthCheckId)]
+}
+
+// desiredRRSet renders one desiredRecord as the record set Route53 should
+// hold. TTL 60, type A, plain value (never an AliasTarget), latency routing
+// keyed on the box label, bound to the box's EIP health check.
+func desiredRRSet(rec desiredRecord, checkID string) r53types.ResourceRecordSet {
+ return r53types.ResourceRecordSet{
+ Name: aws.String(rec.Name),
+ Type: r53types.RRTypeA,
+ SetIdentifier: aws.String(rec.Box),
+ Region: r53types.ResourceRecordSetRegion(rec.Region),
+ TTL: aws.Int64(recordTTL),
+ ResourceRecords: []r53types.ResourceRecord{{Value: aws.String(rec.EIP)}},
+ HealthCheckId: aws.String(checkID),
+ }
+}
+
+// sameRRSet reports whether the zone already holds exactly what we want. It
+// is what makes a steady-state pass write NOTHING: every field this converger
+// sets is compared, and every field it never sets is required to be absent
+// (an unexpected Weight or GeoLocation on our key means somebody changed the
+// routing policy underneath us, and we should rewrite it).
+func sameRRSet(have, want r53types.ResourceRecordSet) bool {
+ if !strings.EqualFold(aws.ToString(have.Name), aws.ToString(want.Name)) {
+ return false
+ }
+ if have.Type != want.Type || aws.ToString(have.SetIdentifier) != aws.ToString(want.SetIdentifier) {
+ return false
+ }
+ if have.Region != want.Region || aws.ToInt64(have.TTL) != aws.ToInt64(want.TTL) {
+ return false
+ }
+ if aws.ToString(have.HealthCheckId) != aws.ToString(want.HealthCheckId) {
+ return false
+ }
+ if have.AliasTarget != nil || have.Weight != nil || have.GeoLocation != nil ||
+ have.GeoProximityLocation != nil || have.CidrRoutingConfig != nil ||
+ have.MultiValueAnswer != nil || have.Failover != "" {
+ return false
+ }
+ if len(have.ResourceRecords) != len(want.ResourceRecords) {
+ return false
+ }
+ for i := range have.ResourceRecords {
+ if aws.ToString(have.ResourceRecords[i].Value) != aws.ToString(want.ResourceRecords[i].Value) {
+ return false
+ }
+ }
+ return true
+}
+
+// recordChanges diffs want against the zone.
+//
+// UPSERTs come first in the returned slice and DELETEs last, so that when the
+// changes are chunked, capacity is added before any is taken away.
+//
+// A wanted key that already holds a record we do NOT manage (a foreign
+// latency record at exactly the (name, SetIdentifier) we would create) is
+// ADOPTED by the UPSERT — CREATE would collide with it and there is no third
+// option. It is logged, because taking over a record is the one write here
+// that changes something we did not make.
+func recordChanges(want []desiredRecord, existing []r53types.ResourceRecordSet, checkByEIP map[string]string, ownedIDs map[string]bool) ([]r53types.Change, error) {
+ byKey := make(map[string]r53types.ResourceRecordSet, len(existing))
+ for _, rr := range existing {
+ byKey[recordKey(rr)] = rr
+ }
+
+ wanted := make(map[string]bool, len(want))
+ var upserts []r53types.Change
+ for _, rec := range want {
+ checkID, ok := checkByEIP[rec.EIP]
+ if !ok || checkID == "" {
+ // Unreachable: Converge creates a check for every distinct EIP
+ // before calling this. Loud rather than silently minting a
+ // record with no health check, which would never withdraw.
+ return nil, fmt.Errorf("steering: no health check for %s (record %s / %s)", rec.EIP, rec.Name, rec.Box)
+ }
+ key := wantKey(rec)
+ wanted[key] = true
+ desired := desiredRRSet(rec, checkID)
+ have, present := byKey[key]
+ if present && sameRRSet(have, desired) {
+ continue
+ }
+ if present && !isManaged(have, ownedIDs) {
+ log.Printf("steering: adopting existing record %s (set %s) — it sits at a name/identifier this deployment steers", rec.Name, rec.Box)
+ }
+ upserts = append(upserts, r53types.Change{Action: r53types.ChangeActionUpsert, ResourceRecordSet: &desired})
+ }
+
+ var deletes []r53types.Change
+ for _, rr := range existing {
+ key := recordKey(rr)
+ if wanted[key] {
+ continue
+ }
+ if isManaged(rr, ownedIDs) {
+ set := rr // DELETE must resubmit the record set verbatim
+ deletes = append(deletes, r53types.Change{Action: r53types.ChangeActionDelete, ResourceRecordSet: &set})
+ continue
+ }
+ // Not managed. Say so when it LOOKS like ours but fails the
+ // ownership gate — a latency A record with a SetIdentifier whose
+ // health check we do not own is either an operator's or an orphan
+ // of ours whose check went missing. Either way we do not touch it,
+ // and either way somebody should know it is there.
+ if rr.Type == r53types.RRTypeA && aws.ToString(rr.SetIdentifier) != "" && rr.Region != "" && rr.AliasTarget == nil {
+ log.Printf("steering: leaving latency record %s (set %s) alone: health check %q is not one this deployment owns",
+ aws.ToString(rr.Name), aws.ToString(rr.SetIdentifier), aws.ToString(rr.HealthCheckId))
+ }
+ }
+
+ return append(upserts, deletes...), nil
+}
+
+// applyChanges submits the changes in chunks.
+//
+// Partial failure is explicit: chunk N lands, chunk N+1 fails, the pass
+// returns the error and stops. That is safe because each chunk is
+// independently valid — an UPSERT is idempotent and a DELETE is of a record
+// nothing wants — so the next housekeeper tick re-reads the zone and issues
+// exactly the changes still outstanding. The one thing that must NOT happen
+// after a failed batch is the health-check GC, and Converge returns before
+// reaching it.
+func applyChanges(ctx context.Context, client route53API, zoneID string, changes []r53types.Change) error {
+ if len(changes) == 0 {
+ return nil
+ }
+ for start := 0; start < len(changes); start += changeBatchLimit {
+ end := start + changeBatchLimit
+ if end > len(changes) {
+ end = len(changes)
+ }
+ chunk := changes[start:end]
+ if _, err := client.ChangeResourceRecordSets(ctx, &route53.ChangeResourceRecordSetsInput{
+ HostedZoneId: aws.String(zoneID),
+ ChangeBatch: &r53types.ChangeBatch{Changes: chunk},
+ }); err != nil {
+ return fmt.Errorf("steering: change batch %d-%d of %d in %s: %w", start, end, len(changes), zoneID, err)
+ }
+ for _, c := range chunk {
+ log.Printf("steering: %s %s (set %s)", c.Action, aws.ToString(c.ResourceRecordSet.Name), aws.ToString(c.ResourceRecordSet.SetIdentifier))
+ }
+ }
+ return nil
+}
+
+// ---------------------------------------------------------------------------
+// naming
+// ---------------------------------------------------------------------------
+
+// distinctEIPs returns want's EIPs, sorted, so health checks are created in a
+// stable order.
+func distinctEIPs(want []desiredRecord) []string {
+ seen := make(map[string]bool, len(want))
+ out := make([]string, 0, len(want))
+ for _, rec := range want {
+ if rec.EIP == "" || seen[rec.EIP] {
+ continue
+ }
+ seen[rec.EIP] = true
+ out = append(out, rec.EIP)
+ }
+ sort.Strings(out)
+ return out
+}
+
+// steeringHostname finds the synthetic edge.<apps-domain> name in want and
+// returns it dot-trimmed, for use as a health check's
+// FullyQualifiedDomainName — an SNI/Host value, not a Route53 record name, so
+// the trailing dot comes off.
+//
+// It is the right SNI name because every edge holds a certificate covering
+// it, so the check exercises the same TLS path a real visitor takes rather
+// than one host's certificate in particular.
+//
+// A member route literally NAMED edge.<apps-domain> produces the identical
+// desired records, and desiredState dedupes on (Name, Box) — so this
+// converger only ever sees one such name and needs no special case: whether
+// it came from the synthetic mint or from a real route, it is the same record
+// and the same SNI name.
+func steeringHostname(want []desiredRecord) string {
+ for _, rec := range want {
+ if strings.HasPrefix(rec.Name, "edge.") {
+ return strings.TrimSuffix(rec.Name, ".")
+ }
+ }
+ return ""
+}
+
+// fallbackHostname picks an SNI name for eip when want carries no
+// edge.<apps-domain> record (no apps domain configured — steering.go logs
+// that case): the first wanted record's own name for that EIP. want's order
+// is deterministic (desiredState sorts and dedupes), so the choice is stable
+// across passes.
+func fallbackHostname(want []desiredRecord, eip string) string {
+ for _, rec := range want {
+ if rec.EIP == eip {
+ return strings.TrimSuffix(rec.Name, ".")
+ }
+ }
+ return ""
+}
diff --git a/internal/steering/route53_test.go b/internal/steering/route53_test.go
new file mode 100644
index 000000000..cb82834a3
--- /dev/null
+++ b/internal/steering/route53_test.go
@@ -0,0 +1,818 @@
+package steering
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "sort"
+ "strings"
+ "testing"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/route53"
+ r53types "github.com/aws/aws-sdk-go-v2/service/route53/types"
+)
+
+// ---------------------------------------------------------------------------
+// the fake
+// ---------------------------------------------------------------------------
+
+// fakeR53 is an in-memory Route53 that records every call and APPLIES the
+// changes it is handed, so a second Converge against the same fake sees the
+// state the first one left. No network, ever — the whole point of route53API
+// being an interface.
+//
+// It deliberately reproduces the real API's refusals that matter to this
+// converger: DELETE of a record that is not present errors (Route53's
+// InvalidChangeBatch), ListTagsForResources refuses more than 10 ids, and
+// DeleteHealthCheck of a check still referenced by a record errors
+// (HealthCheckInUse). A fake that is more permissive than the service would
+// let a bug through.
+type fakeR53 struct {
+ records []r53types.ResourceRecordSet
+ checks []r53types.HealthCheck
+ tags map[string][]r53types.Tag // health check id -> tags
+
+ pageSize int // 0 = one page for everything
+ nextID int
+
+ // call counters — the idempotency assertions live on these
+ nListRecords int
+ nChangeRecords int
+ nListChecks int
+ nListTags int
+ nCreateCheck int
+ nDeleteCheck int
+ nChangeTags int
+
+ submitted []r53types.Change // every change submitted, in order
+ createdChecks []*route53.CreateHealthCheckInput
+ deletedChecks []string
+ taggedIDs []string
+
+ failChange error // ChangeResourceRecordSets returns this
+ failTags error // ChangeTagsForResource returns this
+}
+
+func newFake() *fakeR53 { return &fakeR53{tags: map[string][]r53types.Tag{}} }
+
+// writes is the total number of state-changing calls — the number an
+// idempotent second pass must leave at zero.
+func (f *fakeR53) writes() int {
+ return f.nChangeRecords + f.nCreateCheck + f.nDeleteCheck + f.nChangeTags
+}
+
+func rrKey(rr r53types.ResourceRecordSet) string {
+ return strings.ToLower(aws.ToString(rr.Name)) + "|" + string(rr.Type) + "|" + aws.ToString(rr.SetIdentifier)
+}
+
+func (f *fakeR53) ListResourceRecordSets(ctx context.Context, in *route53.ListResourceRecordSetsInput, _ ...func(*route53.Options)) (*route53.ListResourceRecordSetsOutput, error) {
+ f.nListRecords++
+ if aws.ToString(in.HostedZoneId) == "" {
+ return nil, errors.New("fake: no hosted zone id")
+ }
+ start := 0
+ if in.StartRecordName != nil {
+ for i, rr := range f.records {
+ if strings.EqualFold(aws.ToString(rr.Name), aws.ToString(in.StartRecordName)) &&
+ rr.Type == in.StartRecordType &&
+ aws.ToString(rr.SetIdentifier) == aws.ToString(in.StartRecordIdentifier) {
+ start = i
+ break
+ }
+ }
+ }
+ end := len(f.records)
+ if f.pageSize > 0 && start+f.pageSize < end {
+ end = start + f.pageSize
+ }
+ out := &route53.ListResourceRecordSetsOutput{
+ ResourceRecordSets: append([]r53types.ResourceRecordSet(nil), f.records[start:end]...),
+ }
+ if end < len(f.records) {
+ next := f.records[end]
+ out.IsTruncated = true
+ out.NextRecordName = next.Name
+ out.NextRecordType = next.Type
+ out.NextRecordIdentifier = next.SetIdentifier
+ }
+ return out, nil
+}
+
+func (f *fakeR53) ChangeResourceRecordSets(ctx context.Context, in *route53.ChangeResourceRecordSetsInput, _ ...func(*route53.Options)) (*route53.ChangeResourceRecordSetsOutput, error) {
+ f.nChangeRecords++
+ if f.failChange != nil {
+ return nil, f.failChange
+ }
+ if in.ChangeBatch == nil || len(in.ChangeBatch.Changes) == 0 {
+ return nil, errors.New("fake: empty change batch")
+ }
+ for _, c := range in.ChangeBatch.Changes {
+ f.submitted = append(f.submitted, c)
+ rr := *c.ResourceRecordSet
+ key := rrKey(rr)
+ idx := -1
+ for i, have := range f.records {
+ if rrKey(have) == key {
+ idx = i
+ break
+ }
+ }
+ switch c.Action {
+ case r53types.ChangeActionUpsert:
+ if idx >= 0 {
+ f.records[idx] = rr
+ } else {
+ f.records = append(f.records, rr)
+ }
+ case r53types.ChangeActionDelete:
+ if idx < 0 {
+ return nil, fmt.Errorf("fake: InvalidChangeBatch: no such record %s", key)
+ }
+ f.records = append(f.records[:idx], f.records[idx+1:]...)
+ default:
+ return nil, fmt.Errorf("fake: unsupported action %q", c.Action)
+ }
+ }
+ sort.SliceStable(f.records, func(i, j int) bool { return rrKey(f.records[i]) < rrKey(f.records[j]) })
+ return &route53.ChangeResourceRecordSetsOutput{}, nil
+}
+
+func (f *fakeR53) ListHealthChecks(ctx context.Context, in *route53.ListHealthChecksInput, _ ...func(*route53.Options)) (*route53.ListHealthChecksOutput, error) {
+ f.nListChecks++
+ start := 0
+ if in.Marker != nil {
+ for i, hc := range f.checks {
+ if aws.ToString(hc.Id) == aws.ToString(in.Marker) {
+ start = i
+ break
+ }
+ }
+ }
+ end := len(f.checks)
+ if f.pageSize > 0 && start+f.pageSize < end {
+ end = start + f.pageSize
+ }
+ out := &route53.ListHealthChecksOutput{
+ HealthChecks: append([]r53types.HealthCheck(nil), f.checks[start:end]...),
+ }
+ if end < len(f.checks) {
+ out.IsTruncated = true
+ out.NextMarker = f.checks[end].Id
+ }
+ return out, nil
+}
+
+func (f *fakeR53) ListTagsForResources(ctx context.Context, in *route53.ListTagsForResourcesInput, _ ...func(*route53.Options)) (*route53.ListTagsForResourcesOutput, error) {
+ f.nListTags++
+ if in.ResourceType != r53types.TagResourceTypeHealthcheck {
+ return nil, fmt.Errorf("fake: unexpected resource type %q", in.ResourceType)
+ }
+ if len(in.ResourceIds) == 0 || len(in.ResourceIds) > 10 {
+ return nil, fmt.Errorf("fake: ListTagsForResources takes 1-10 ids, got %d", len(in.ResourceIds))
+ }
+ out := &route53.ListTagsForResourcesOutput{}
+ for _, id := range in.ResourceIds {
+ out.ResourceTagSets = append(out.ResourceTagSets, r53types.ResourceTagSet{
+ ResourceId: aws.String(id),
+ ResourceType: r53types.TagResourceTypeHealthcheck,
+ Tags: f.tags[id],
+ })
+ }
+ return out, nil
+}
+
+func (f *fakeR53) CreateHealthCheck(ctx context.Context, in *route53.CreateHealthCheckInput, _ ...func(*route53.Options)) (*route53.CreateHealthCheckOutput, error) {
+ f.nCreateCheck++
+ f.createdChecks = append(f.createdChecks, in)
+ f.nextID++
+ id := fmt.Sprintf("hc-%d", f.nextID)
+ hc := r53types.HealthCheck{
+ Id: aws.String(id),
+ CallerReference: in.CallerReference,
+ HealthCheckConfig: in.HealthCheckConfig,
+ }
+ f.checks = append(f.checks, hc)
+ return &route53.CreateHealthCheckOutput{HealthCheck: &hc, Location: aws.String("https://route53/" + id)}, nil
+}
+
+func (f *fakeR53) DeleteHealthCheck(ctx context.Context, in *route53.DeleteHealthCheckInput, _ ...func(*route53.Options)) (*route53.DeleteHealthCheckOutput, error) {
+ f.nDeleteCheck++
+ id := aws.ToString(in.HealthCheckId)
+ for _, rr := range f.records {
+ if aws.ToString(rr.HealthCheckId) == id {
+ return nil, fmt.Errorf("fake: HealthCheckInUse: %s", id)
+ }
+ }
+ f.deletedChecks = append(f.deletedChecks, id)
+ for i, hc := range f.checks {
+ if aws.ToString(hc.Id) == id {
+ f.checks = append(f.checks[:i], f.checks[i+1:]...)
+ break
+ }
+ }
+ delete(f.tags, id)
+ return &route53.DeleteHealthCheckOutput{}, nil
+}
+
+func (f *fakeR53) ChangeTagsForResource(ctx context.Context, in *route53.ChangeTagsForResourceInput, _ ...func(*route53.Options)) (*route53.ChangeTagsForResourceOutput, error) {
+ f.nChangeTags++
+ if f.failTags != nil {
+ return nil, f.failTags
+ }
+ id := aws.ToString(in.ResourceId)
+ f.taggedIDs = append(f.taggedIDs, id)
+ f.tags[id] = append(f.tags[id], in.AddTags...)
+ return &route53.ChangeTagsForResourceOutput{}, nil
+}
+
+// ---------------------------------------------------------------------------
+// fixtures + helpers
+// ---------------------------------------------------------------------------
+
+const testZone = "Z0STEERINGTEST"
+
+var (
+ dublinRec = desiredRecord{Name: "edge.example.com.", EIP: "1.2.3.4", Box: "dublin", Region: "eu-west-1"}
+ oregonRec = desiredRecord{Name: "edge.example.com.", EIP: "5.6.7.8", Box: "oregon", Region: "us-west-2"}
+)
+
+// hostRecs mints the per-host half of a desired set: one record per edge.
+func hostRecs(name string, edges ...desiredRecord) []desiredRecord {
+ out := make([]desiredRecord, 0, len(edges))
+ for _, e := range edges {
+ out = append(out, desiredRecord{Name: name, EIP: e.EIP, Box: e.Box, Region: e.Region})
+ }
+ return out
+}
+
+func staticA(name, ip string) r53types.ResourceRecordSet {
+ return r53types.ResourceRecordSet{
+ Name: aws.String(name),
+ Type: r53types.RRTypeA,
+ TTL: aws.Int64(300),
+ ResourceRecords: []r53types.ResourceRecord{{Value: aws.String(ip)}},
+ }
+}
+
+func recordNames(recs []r53types.ResourceRecordSet) []string {
+ out := make([]string, 0, len(recs))
+ for _, rr := range recs {
+ out = append(out, rrKey(rr))
+ }
+ sort.Strings(out)
+ return out
+}
+
+func find(t *testing.T, f *fakeR53, name, setID string) r53types.ResourceRecordSet {
+ t.Helper()
+ for _, rr := range f.records {
+ if strings.EqualFold(aws.ToString(rr.Name), name) && aws.ToString(rr.SetIdentifier) == setID {
+ return rr
+ }
+ }
+ t.Fatalf("no record %s / setid %q in zone; have %v", name, setID, recordNames(f.records))
+ return r53types.ResourceRecordSet{}
+}
+
+func mustConverge(t *testing.T, f *fakeR53, want []desiredRecord) {
+ t.Helper()
+ if err := Converge(context.Background(), f, testZone, want); err != nil {
+ t.Fatalf("Converge: %v", err)
+ }
+}
+
+// ---------------------------------------------------------------------------
+// tests
+// ---------------------------------------------------------------------------
+
+// TestConvergeCreateFromEmpty is the brief's create-from-empty case: an
+// empty zone, two armed edges, one steered host.
+func TestConvergeCreateFromEmpty(t *testing.T) {
+ f := newFake()
+ want := append([]desiredRecord{dublinRec, oregonRec}, hostRecs("app.example.com.", dublinRec, oregonRec)...)
+ mustConverge(t, f, want)
+
+ if len(f.records) != 4 {
+ t.Fatalf("want 4 records, got %d: %v", len(f.records), recordNames(f.records))
+ }
+ if f.nCreateCheck != 2 {
+ t.Fatalf("want one health check per distinct EIP (2), got %d", f.nCreateCheck)
+ }
+ if f.nChangeTags != 2 {
+ t.Fatalf("want every created check tagged (2), got %d", f.nChangeTags)
+ }
+
+ // health check shape
+ byIP := map[string]*route53.CreateHealthCheckInput{}
+ for _, in := range f.createdChecks {
+ byIP[aws.ToString(in.HealthCheckConfig.IPAddress)] = in
+ }
+ for _, ip := range []string{"1.2.3.4", "5.6.7.8"} {
+ in, ok := byIP[ip]
+ if !ok {
+ t.Fatalf("no health check created for %s", ip)
+ }
+ cfg := in.HealthCheckConfig
+ if cfg.Type != r53types.HealthCheckTypeHttps {
+ t.Errorf("%s: type = %q, want HTTPS", ip, cfg.Type)
+ }
+ if aws.ToInt32(cfg.Port) != 443 {
+ t.Errorf("%s: port = %d, want 443", ip, aws.ToInt32(cfg.Port))
+ }
+ if !aws.ToBool(cfg.EnableSNI) {
+ t.Errorf("%s: EnableSNI false, want true (the SNI name is the whole point)", ip)
+ }
+ // FQDN = the steering hostname, dot-trimmed (it is an SNI/Host value,
+ // not a Route53 record name).
+ if got := aws.ToString(cfg.FullyQualifiedDomainName); got != "edge.example.com" {
+ t.Errorf("%s: FQDN = %q, want the steering hostname edge.example.com", ip, got)
+ }
+ if !strings.HasPrefix(aws.ToString(in.CallerReference), "carlos-steering-") {
+ t.Errorf("%s: CallerReference = %q, want a carlos-steering- prefix", ip, aws.ToString(in.CallerReference))
+ }
+ if n := len(aws.ToString(in.CallerReference)); n > 64 {
+ t.Errorf("%s: CallerReference is %d chars, Route53 caps it at 64", ip, n)
+ }
+ }
+
+ // every created check carries the zone-scoped tag
+ for id, tags := range f.tags {
+ var found bool
+ for _, tg := range tags {
+ if aws.ToString(tg.Key) == steeringTagKey && aws.ToString(tg.Value) == testZone {
+ found = true
+ }
+ }
+ if !found {
+ t.Errorf("check %s not tagged %s=%s", id, steeringTagKey, testZone)
+ }
+ }
+
+ // record shape
+ rr := find(t, f, "app.example.com.", "dublin")
+ if rr.Type != r53types.RRTypeA {
+ t.Errorf("type = %q, want A", rr.Type)
+ }
+ if aws.ToInt64(rr.TTL) != 60 {
+ t.Errorf("TTL = %d, want 60", aws.ToInt64(rr.TTL))
+ }
+ if string(rr.Region) != "eu-west-1" {
+ t.Errorf("Region = %q, want eu-west-1", rr.Region)
+ }
+ if rr.AliasTarget != nil {
+ t.Errorf("AliasTarget set; these are plain latency records to EIPs")
+ }
+ if len(rr.ResourceRecords) != 1 || aws.ToString(rr.ResourceRecords[0].Value) != "1.2.3.4" {
+ t.Errorf("value = %v, want 1.2.3.4", rr.ResourceRecords)
+ }
+ if aws.ToString(rr.HealthCheckId) == "" {
+ t.Errorf("record not bound to a health check")
+ }
+ // the record for the same EIP under a different name shares the check
+ edgeRR := find(t, f, "edge.example.com.", "dublin")
+ if aws.ToString(edgeRR.HealthCheckId) != aws.ToString(rr.HealthCheckId) {
+ t.Errorf("one check per EIP violated: %q vs %q", aws.ToString(edgeRR.HealthCheckId), aws.ToString(rr.HealthCheckId))
+ }
+}
+
+// TestConvergeIdempotentZeroWrites is the brief's headline safety property:
+// a second pass with an identical want makes NO write calls at all.
+func TestConvergeIdempotentZeroWrites(t *testing.T) {
+ f := newFake()
+ want := append([]desiredRecord{dublinRec, oregonRec}, hostRecs("app.example.com.", dublinRec, oregonRec)...)
+ mustConverge(t, f, want)
+
+ before := f.writes()
+ if before == 0 {
+ t.Fatalf("first pass made no writes; the test proves nothing")
+ }
+ mustConverge(t, f, want)
+ if got := f.writes() - before; got != 0 {
+ t.Fatalf("second pass made %d write calls, want 0 (records=%d creates=%d deletes=%d tags=%d)",
+ got, f.nChangeRecords, f.nCreateCheck, f.nDeleteCheck, f.nChangeTags)
+ }
+ // a third pass, for good measure — no slow drift
+ mustConverge(t, f, want)
+ if got := f.writes() - before; got != 0 {
+ t.Fatalf("third pass made %d write calls, want 0", got)
+ }
+}
+
+// TestConvergeAddEdge: a second edge arms. Only the new records are written;
+// the existing ones are left alone.
+func TestConvergeAddEdge(t *testing.T) {
+ f := newFake()
+ one := append([]desiredRecord{dublinRec}, hostRecs("app.example.com.", dublinRec)...)
+ mustConverge(t, f, one)
+ base := f.writes()
+ f.submitted = nil
+
+ two := append([]desiredRecord{dublinRec, oregonRec}, hostRecs("app.example.com.", dublinRec, oregonRec)...)
+ mustConverge(t, f, two)
+
+ if f.nCreateCheck != 2 {
+ t.Fatalf("want exactly one NEW check (2 total), got %d total", f.nCreateCheck)
+ }
+ if f.nDeleteCheck != 0 {
+ t.Fatalf("adding an edge deleted %d checks", f.nDeleteCheck)
+ }
+ if len(f.submitted) != 2 {
+ t.Fatalf("want 2 changes (the new edge's records only), got %d: %v", len(f.submitted), f.submitted)
+ }
+ for _, c := range f.submitted {
+ if c.Action != r53types.ChangeActionUpsert {
+ t.Errorf("adding an edge submitted a %s", c.Action)
+ }
+ if aws.ToString(c.ResourceRecordSet.SetIdentifier) != "oregon" {
+ t.Errorf("rewrote an untouched record: %s", rrKey(*c.ResourceRecordSet))
+ }
+ }
+ if len(f.records) != 4 {
+ t.Fatalf("want 4 records, got %v", recordNames(f.records))
+ }
+ _ = base
+}
+
+// TestConvergeRemoveEdge: an edge disarms. Its records go, and the health
+// check it alone used is GC'd; the surviving edge's check is retained.
+func TestConvergeRemoveEdge(t *testing.T) {
+ f := newFake()
+ two := append([]desiredRecord{dublinRec, oregonRec}, hostRecs("app.example.com.", dublinRec, oregonRec)...)
+ mustConverge(t, f, two)
+ oregonCheck := aws.ToString(find(t, f, "edge.example.com.", "oregon").HealthCheckId)
+ dublinCheck := aws.ToString(find(t, f, "edge.example.com.", "dublin").HealthCheckId)
+ f.submitted = nil
+
+ one := append([]desiredRecord{dublinRec}, hostRecs("app.example.com.", dublinRec)...)
+ mustConverge(t, f, one)
+
+ if len(f.records) != 2 {
+ t.Fatalf("want 2 records left, got %v", recordNames(f.records))
+ }
+ for _, rr := range f.records {
+ if aws.ToString(rr.SetIdentifier) == "oregon" {
+ t.Errorf("oregon record survived: %s", rrKey(rr))
+ }
+ }
+ for _, c := range f.submitted {
+ if c.Action != r53types.ChangeActionDelete {
+ t.Errorf("removing an edge submitted a %s for %s", c.Action, rrKey(*c.ResourceRecordSet))
+ }
+ }
+ if len(f.deletedChecks) != 1 || f.deletedChecks[0] != oregonCheck {
+ t.Fatalf("want the orphaned check %s GC'd, got %v", oregonCheck, f.deletedChecks)
+ }
+ // the surviving check is untouched and still bound
+ if got := aws.ToString(find(t, f, "app.example.com.", "dublin").HealthCheckId); got != dublinCheck {
+ t.Errorf("dublin's check changed: %q -> %q", dublinCheck, got)
+ }
+}
+
+// TestConvergeRemoveSteering: a host stops being steered. Its records go,
+// but the checks stay — another host (and the steering hostname) still uses
+// the same EIPs.
+func TestConvergeRemoveSteering(t *testing.T) {
+ f := newFake()
+ both := append([]desiredRecord{dublinRec, oregonRec}, hostRecs("app.example.com.", dublinRec, oregonRec)...)
+ both = append(both, hostRecs("other.example.com.", dublinRec, oregonRec)...)
+ mustConverge(t, f, both)
+ if len(f.records) != 6 {
+ t.Fatalf("setup: want 6 records, got %v", recordNames(f.records))
+ }
+ checksBefore := len(f.checks)
+ f.submitted = nil
+
+ one := append([]desiredRecord{dublinRec, oregonRec}, hostRecs("other.example.com.", dublinRec, oregonRec)...)
+ mustConverge(t, f, one)
+
+ if len(f.records) != 4 {
+ t.Fatalf("want 4 records left, got %v", recordNames(f.records))
+ }
+ for _, rr := range f.records {
+ if strings.EqualFold(aws.ToString(rr.Name), "app.example.com.") {
+ t.Errorf("unsteered host's record survived: %s", rrKey(rr))
+ }
+ }
+ if len(f.deletedChecks) != 0 {
+ t.Fatalf("checks still in use were GC'd: %v", f.deletedChecks)
+ }
+ if len(f.checks) != checksBefore {
+ t.Fatalf("check count changed %d -> %d", checksBefore, len(f.checks))
+ }
+}
+
+// TestConvergeStaticRecordsUntouched is the deletion-safety test. The zone is
+// seeded with everything an operator's zone really holds; Converge must
+// submit no change that touches any of it.
+func TestConvergeStaticRecordsUntouched(t *testing.T) {
+ f := newFake()
+ f.records = []r53types.ResourceRecordSet{
+ staticA("example.com.", "9.9.9.9"), // apex A, no SetIdentifier
+ staticA("www.example.com.", "9.9.9.9"), // plain A
+ staticA("app.example.com.", "9.9.9.9"), // plain A sharing a STEERED name
+ {Name: aws.String("example.com."), Type: r53types.RRTypeNs, TTL: aws.Int64(172800)},
+ {Name: aws.String("example.com."), Type: r53types.RRTypeSoa, TTL: aws.Int64(900)},
+ {Name: aws.String("mail.example.com."), Type: r53types.RRTypeCname, TTL: aws.Int64(300)},
+ { // a WEIGHTED A with a SetIdentifier — has an identifier but is not latency
+ Name: aws.String("weighted.example.com."), Type: r53types.RRTypeA,
+ SetIdentifier: aws.String("dublin"), Weight: aws.Int64(10), TTL: aws.Int64(60),
+ ResourceRecords: []r53types.ResourceRecord{{Value: aws.String("8.8.8.8")}},
+ },
+ { // a LATENCY A bound to a health check that is not ours
+ Name: aws.String("legacy.example.com."), Type: r53types.RRTypeA,
+ SetIdentifier: aws.String("dublin"), Region: "eu-west-1", TTL: aws.Int64(60),
+ HealthCheckId: aws.String("hc-someone-else"),
+ ResourceRecords: []r53types.ResourceRecord{{Value: aws.String("7.7.7.7")}},
+ },
+ { // a LATENCY ALIAS record — never a shape this converger writes
+ Name: aws.String("alias.example.com."), Type: r53types.RRTypeA,
+ SetIdentifier: aws.String("dublin"), Region: "eu-west-1",
+ AliasTarget: &r53types.AliasTarget{DNSName: aws.String("elb.example.com.")},
+ },
+ }
+ before := recordNames(f.records)
+
+ want := append([]desiredRecord{dublinRec}, hostRecs("app.example.com.", dublinRec)...)
+ mustConverge(t, f, want)
+
+ for _, c := range f.submitted {
+ if c.Action == r53types.ChangeActionDelete {
+ t.Errorf("submitted a DELETE for %s — static/foreign records are out of bounds", rrKey(*c.ResourceRecordSet))
+ }
+ if aws.ToString(c.ResourceRecordSet.SetIdentifier) == "" {
+ t.Errorf("submitted a change for a record with no SetIdentifier: %s", rrKey(*c.ResourceRecordSet))
+ }
+ }
+ for _, name := range before {
+ var still bool
+ for _, have := range recordNames(f.records) {
+ if have == name {
+ still = true
+ }
+ }
+ if !still {
+ t.Errorf("record %s disappeared", name)
+ }
+ }
+ // the plain A for the steered name still stands beside the latency ones
+ // (different (name,type,SetIdentifier) keys — no collision).
+ plain := find(t, f, "app.example.com.", "")
+ if aws.ToInt64(plain.TTL) != 300 {
+ t.Errorf("the static app.example.com. A record was rewritten: TTL %d", aws.ToInt64(plain.TTL))
+ }
+}
+
+// TestConvergeHealthCheckGCIsTagScoped: a check must carry OUR tag, with OUR
+// zone id, before it can ever be deleted. Everything else in the account is
+// invisible to the GC.
+func TestConvergeHealthCheckGCIsTagScoped(t *testing.T) {
+ f := newFake()
+ // three checks on an EIP nobody wants any more
+ f.checks = []r53types.HealthCheck{
+ {Id: aws.String("hc-untagged"), HealthCheckConfig: &r53types.HealthCheckConfig{
+ Type: r53types.HealthCheckTypeHttps, Port: aws.Int32(443), IPAddress: aws.String("10.0.0.1")}},
+ {Id: aws.String("hc-other-zone"), HealthCheckConfig: &r53types.HealthCheckConfig{
+ Type: r53types.HealthCheckTypeHttps, Port: aws.Int32(443), IPAddress: aws.String("10.0.0.2")}},
+ {Id: aws.String("hc-ours"), HealthCheckConfig: &r53types.HealthCheckConfig{
+ Type: r53types.HealthCheckTypeHttps, Port: aws.Int32(443), IPAddress: aws.String("10.0.0.3")}},
+ {Id: aws.String("hc-other-service"), HealthCheckConfig: &r53types.HealthCheckConfig{
+ Type: r53types.HealthCheckTypeHttp, Port: aws.Int32(80), IPAddress: aws.String("10.0.0.4")}},
+ }
+ f.tags["hc-other-zone"] = []r53types.Tag{{Key: aws.String(steeringTagKey), Value: aws.String("ZSOMEOTHERZONE")}}
+ f.tags["hc-ours"] = []r53types.Tag{{Key: aws.String(steeringTagKey), Value: aws.String(testZone)}}
+ f.tags["hc-other-service"] = []r53types.Tag{{Key: aws.String("Name"), Value: aws.String("someone else's check")}}
+
+ mustConverge(t, f, []desiredRecord{dublinRec})
+
+ if len(f.deletedChecks) != 1 || f.deletedChecks[0] != "hc-ours" {
+ t.Fatalf("GC deleted %v; only the tagged-for-THIS-zone check may ever go", f.deletedChecks)
+ }
+}
+
+// TestConvergeEmptyWantRefusesTeardown: an empty desired set is a degraded
+// read (zero armed edges), not an instruction to delete production DNS.
+func TestConvergeEmptyWantRefusesTeardown(t *testing.T) {
+ f := newFake()
+ want := append([]desiredRecord{dublinRec}, hostRecs("app.example.com.", dublinRec)...)
+ mustConverge(t, f, want)
+ before := f.writes()
+ recs := len(f.records)
+
+ buf := captureLog(t)
+ if err := Converge(context.Background(), f, testZone, nil); err != nil {
+ t.Fatalf("empty want should not error: %v", err)
+ }
+ if f.writes() != before {
+ t.Fatalf("empty want made %d write calls", f.writes()-before)
+ }
+ if len(f.records) != recs {
+ t.Fatalf("empty want removed records: %v", recordNames(f.records))
+ }
+ if len(f.deletedChecks) != 0 {
+ t.Fatalf("empty want GC'd checks: %v", f.deletedChecks)
+ }
+ if !strings.Contains(buf.String(), "refusing") {
+ t.Errorf("the refusal was silent; log = %q", buf.String())
+ }
+}
+
+// TestConvergeRejectsBadZoneID: a malformed or empty zone id never reaches
+// the API — a typo must not become a call against some other zone.
+func TestConvergeRejectsBadZoneID(t *testing.T) {
+ for _, zone := range []string{"", " ", "z0abc", "/hostedzone/Z0ABC", "Z0abc", "NOTAZONE"} {
+ f := newFake()
+ err := Converge(context.Background(), f, zone, []desiredRecord{dublinRec})
+ if err == nil {
+ t.Errorf("zone %q accepted", zone)
+ }
+ if f.nListRecords+f.nListChecks+f.writes() != 0 {
+ t.Errorf("zone %q reached the API", zone)
+ }
+ }
+}
+
+// TestConvergeHealthCheckFQDNFallback: with no edge.<apps-domain> record in
+// want, the check's SNI name falls back to a record's own name.
+func TestConvergeHealthCheckFQDNFallback(t *testing.T) {
+ f := newFake()
+ mustConverge(t, f, hostRecs("app.example.com.", dublinRec))
+
+ if len(f.createdChecks) != 1 {
+ t.Fatalf("want 1 check, got %d", len(f.createdChecks))
+ }
+ if got := aws.ToString(f.createdChecks[0].HealthCheckConfig.FullyQualifiedDomainName); got != "app.example.com" {
+ t.Fatalf("FQDN = %q, want the record's own name app.example.com", got)
+ }
+}
+
+// TestConvergePagination: both listings paginate, and a converger that
+// stopped at page one would see an empty zone and rewrite everything.
+func TestConvergePagination(t *testing.T) {
+ f := newFake()
+ want := append([]desiredRecord{dublinRec, oregonRec}, hostRecs("app.example.com.", dublinRec, oregonRec)...)
+ want = append(want, hostRecs("b.example.com.", dublinRec, oregonRec)...)
+ want = append(want, hostRecs("c.example.com.", dublinRec, oregonRec)...)
+ mustConverge(t, f, want)
+
+ f.pageSize = 1 // force many pages on the second pass
+ before := f.writes()
+ mustConverge(t, f, want)
+ if got := f.writes() - before; got != 0 {
+ t.Fatalf("paginated pass made %d writes, want 0 — a page was missed", got)
+ }
+ if f.nListRecords < 3 || f.nListChecks < 2 {
+ t.Fatalf("pagination not exercised: %d record pages, %d check pages", f.nListRecords, f.nListChecks)
+ }
+}
+
+// TestConvergeTagFailureRemovesTheOrphan: a check created but not tagged can
+// never be reused (untagged = not ours) and never GC'd — an unbounded leak,
+// one per pass. It is deleted again before the pass fails.
+func TestConvergeTagFailureRemovesTheOrphan(t *testing.T) {
+ f := newFake()
+ f.failTags = errors.New("boom")
+ err := Converge(context.Background(), f, testZone, []desiredRecord{dublinRec})
+ if err == nil {
+ t.Fatalf("tagging failure did not fail the pass")
+ }
+ if len(f.deletedChecks) != 1 {
+ t.Fatalf("the untaggable check was leaked: created %d, deleted %v", f.nCreateCheck, f.deletedChecks)
+ }
+ if f.nChangeRecords != 0 {
+ t.Errorf("records were written despite the failed pass")
+ }
+}
+
+// TestConvergeChangeFailureIsLoud: a failed batch fails the pass. The
+// housekeeper retries next tick; it must not be swallowed.
+func TestConvergeChangeFailureIsLoud(t *testing.T) {
+ f := newFake()
+ f.failChange = errors.New("InvalidChangeBatch")
+ err := Converge(context.Background(), f, testZone, []desiredRecord{dublinRec})
+ if err == nil {
+ t.Fatalf("a failed change batch did not fail the pass")
+ }
+ if !strings.Contains(err.Error(), "InvalidChangeBatch") {
+ t.Errorf("the underlying error was not wrapped: %v", err)
+ }
+ // nothing may be GC'd on a pass whose record changes did not land
+ if len(f.deletedChecks) != 0 {
+ t.Errorf("checks GC'd after a failed record batch: %v", f.deletedChecks)
+ }
+}
+
+// TestConvergeBatchesChanges: a large desired set is chunked, and every
+// chunk is within the batch limit.
+func TestConvergeBatchesChanges(t *testing.T) {
+ f := newFake()
+ var want []desiredRecord
+ for i := 0; i < changeBatchLimit+5; i++ {
+ want = append(want, desiredRecord{
+ Name: fmt.Sprintf("h%03d.example.com.", i), EIP: "1.2.3.4", Box: "dublin", Region: "eu-west-1",
+ })
+ }
+ mustConverge(t, f, want)
+ if f.nChangeRecords != 2 {
+ t.Fatalf("want 2 batches for %d changes, got %d", len(want), f.nChangeRecords)
+ }
+ if len(f.records) != len(want) {
+ t.Fatalf("want %d records, got %d", len(want), len(f.records))
+ }
+}
+
+// TestConvergeAdoptsForeignLatencyRecordUnderOurKey: a latency record already
+// sitting at a (name, SetIdentifier) we want is taken over by UPSERT — it is
+// exactly the record we would otherwise create, and CREATE would collide.
+func TestConvergeAdoptsForeignLatencyRecordUnderOurKey(t *testing.T) {
+ f := newFake()
+ f.records = []r53types.ResourceRecordSet{{
+ Name: aws.String("edge.example.com."), Type: r53types.RRTypeA,
+ SetIdentifier: aws.String("dublin"), Region: "eu-west-1", TTL: aws.Int64(300),
+ ResourceRecords: []r53types.ResourceRecord{{Value: aws.String("99.99.99.99")}},
+ }}
+ buf := captureLog(t)
+ mustConverge(t, f, []desiredRecord{dublinRec})
+
+ rr := find(t, f, "edge.example.com.", "dublin")
+ if aws.ToString(rr.ResourceRecords[0].Value) != "1.2.3.4" || aws.ToInt64(rr.TTL) != 60 {
+ t.Fatalf("record not converged: %v TTL=%d", rr.ResourceRecords, aws.ToInt64(rr.TTL))
+ }
+ if !strings.Contains(buf.String(), "adopting") {
+ t.Errorf("the takeover was silent; log = %q", buf.String())
+ }
+}
+
+// TestIsManagedGates pins the deletion boundary GATE BY GATE.
+//
+// The zone-level test above (TestConvergeStaticRecordsUntouched) proves the
+// boundary holds for the records a real zone contains, but its fixtures fail
+// several gates at once — so it cannot tell which gate is doing the work, and
+// stays green if one of them is deleted. This table starts from a record that
+// passes EVERY gate and breaks exactly one at a time. Each row must flip the
+// verdict to unmanaged on its own.
+func TestIsManagedGates(t *testing.T) {
+ owned := map[string]bool{"hc-ours": true}
+ managed := func() r53types.ResourceRecordSet {
+ return r53types.ResourceRecordSet{
+ Name: aws.String("app.example.com."), Type: r53types.RRTypeA,
+ SetIdentifier: aws.String("dublin"), Region: "eu-west-1", TTL: aws.Int64(60),
+ HealthCheckId: aws.String("hc-ours"),
+ ResourceRecords: []r53types.ResourceRecord{{Value: aws.String("1.2.3.4")}},
+ }
+ }
+ if !isManaged(managed(), owned) {
+ t.Fatalf("the baseline record is not managed; the table below proves nothing")
+ }
+
+ tests := []struct {
+ name string
+ break_ func(*r53types.ResourceRecordSet)
+ }{
+ {"not an A record", func(rr *r53types.ResourceRecordSet) { rr.Type = r53types.RRTypeAaaa }},
+ {"no SetIdentifier (static/operator territory)", func(rr *r53types.ResourceRecordSet) { rr.SetIdentifier = nil }},
+ {"empty SetIdentifier", func(rr *r53types.ResourceRecordSet) { rr.SetIdentifier = aws.String("") }},
+ {"not a latency record", func(rr *r53types.ResourceRecordSet) { rr.Region = "" }},
+ {"an alias record", func(rr *r53types.ResourceRecordSet) {
+ rr.AliasTarget = &r53types.AliasTarget{DNSName: aws.String("elb.example.com.")}
+ }},
+ {"health check we do not own", func(rr *r53types.ResourceRecordSet) { rr.HealthCheckId = aws.String("hc-theirs") }},
+ {"no health check at all", func(rr *r53types.ResourceRecordSet) { rr.HealthCheckId = nil }},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ rr := managed()
+ tt.break_(&rr)
+ if isManaged(rr, owned) {
+ t.Fatalf("record is still managed — this gate does not stand on its own")
+ }
+ })
+ }
+
+ // and the empty-ownership case: if the tag listing found nothing ours,
+ // NOTHING in the zone is deletable.
+ if isManaged(managed(), map[string]bool{}) {
+ t.Errorf("a record is managed with no owned health checks at all")
+ }
+}
+
+// TestSteeringZoneIDFromEnv pins the dormancy contract Task 6 consumes.
+func TestSteeringZoneIDFromEnv(t *testing.T) {
+ tests := []struct {
+ set string
+ want string
+ }{
+ {"", ""},
+ {" ", ""},
+ {"Z0STEERINGTEST", "Z0STEERINGTEST"},
+ {" Z0STEERINGTEST ", "Z0STEERINGTEST"},
+ {"/hostedzone/Z0STEERINGTEST", ""},
+ {"z0steeringtest", ""},
+ }
+ for _, tt := range tests {
+ t.Setenv(steeringZoneEnv, tt.set)
+ if got := SteeringZoneID(); got != tt.want {
+ t.Errorf("SteeringZoneID() with %q = %q, want %q", tt.set, got, tt.want)
+ }
+ }
+}