| 1 | package steering |
| 2 | |
| 3 | // route53.go — the converger. This is the ONLY file in the tree that holds |
| 4 | // write credentials to a production DNS zone, so its whole design is about |
| 5 | // what it REFUSES to touch. Read the managed-set rule (isManaged) and the |
| 6 | // health-check ownership rule (ownedHealthChecks) before changing anything |
| 7 | // here: everything else is bookkeeping around those two boundaries. |
| 8 | // |
| 9 | // converge is UNEXPORTED: ConvergeDeployment (deployment.go) is this |
| 10 | // package's only entry point, so no caller outside it can converge a desired |
| 11 | // set the fold did not build. It is reached, from Task 6's housekeeper duty |
| 12 | // (internal/host/steering_loop.go) — but only on a deployment that sets |
| 13 | // CARLOS_STEERING_ZONE_ID, which no box does today: unset keeps the duty |
| 14 | // dormant, zero API calls, zero writes. |
| 15 | |
| 16 | import ( |
| 17 | "context" |
| 18 | "fmt" |
| 19 | "log" |
| 20 | "os" |
| 21 | "regexp" |
| 22 | "sort" |
| 23 | "strings" |
| 24 | "time" |
| 25 | |
| 26 | "github.com/aws/aws-sdk-go-v2/aws" |
| 27 | awsconfig "github.com/aws/aws-sdk-go-v2/config" |
| 28 | "github.com/aws/aws-sdk-go-v2/service/route53" |
| 29 | r53types "github.com/aws/aws-sdk-go-v2/service/route53/types" |
| 30 | "github.com/carlosframework/platform/internal/rootaws" |
| 31 | ) |
| 32 | |
| 33 | const ( |
| 34 | // steeringZoneEnv names the deployment's steering hosted zone. Unset = |
| 35 | // dormant, which is every box today. |
| 36 | steeringZoneEnv = "CARLOS_STEERING_ZONE_ID" |
| 37 | |
| 38 | // steeringTagKey / its value (the zone id) is the ONLY thing that makes |
| 39 | // a Route53 health check eligible for deletion by this code. An untagged |
| 40 | // check, or one tagged for another deployment's zone, is invisible to |
| 41 | // the GC — see ownedHealthChecks. |
| 42 | steeringTagKey = "carlos-steering" |
| 43 | |
| 44 | // recordTTL is short on purpose: the health check withdraws a dead |
| 45 | // edge's answer in ~90s, and a long TTL would keep resolvers handing it |
| 46 | // out for far longer than that. |
| 47 | recordTTL = 60 |
| 48 | |
| 49 | // health check shape. HTTPS on 443 with SNI: the name in |
| 50 | // FullyQualifiedDomainName is what the edge sees in the handshake, so it |
| 51 | // must be a name the edge holds a certificate for. |
| 52 | healthCheckPort = 443 |
| 53 | healthCheckPath = "/" |
| 54 | healthCheckInterval = 30 // seconds |
| 55 | healthCheckFailureThresh = 3 // 3 x 30s ~= the spec's 90s withdrawal |
| 56 | healthCheckTagBatch = 10 // ListTagsForResources' own cap |
| 57 | healthCheckCallerRefPrefix = "carlos-steering-" |
| 58 | |
| 59 | // changeBatchLimit chunks ChangeResourceRecordSets well under Route53's |
| 60 | // own 1000-change cap. Smaller batches mean a failure retries less work |
| 61 | // and names a smaller blast radius in the log. |
| 62 | changeBatchLimit = 100 |
| 63 | |
| 64 | // shrinkFloor is the number of managed records a pass may always remove, |
| 65 | // however small the zone. Without it, the proportional guard below would |
| 66 | // make a 2-record zone unable to lose a single record. See shrinkLimit. |
| 67 | shrinkFloor = 2 |
| 68 | ) |
| 69 | |
| 70 | // steeringZonePattern is the region-host tofu module's own zone-id shape |
| 71 | // (infra/modules/region-host/variables.tf: `^Z[A-Z0-9]+$`) — a BARE id, never |
| 72 | // the "/hostedzone/Z..." ARN-ish form the API returns in some places. A typo |
| 73 | // that slips this gate would be a write against some other zone, so the gate |
| 74 | // runs before any client call, not after. |
| 75 | var steeringZonePattern = regexp.MustCompile(`^Z[A-Z0-9]+$`) |
| 76 | |
| 77 | // route53API is the slice of the Route53 client this converger uses — an |
| 78 | // interface so tests substitute an in-memory fake and never reach AWS |
| 79 | // (route53_test.go). Same seam as internal/edge/route53dns.go's own |
| 80 | // route53API. |
| 81 | type route53API interface { |
| 82 | ListResourceRecordSets(ctx context.Context, in *route53.ListResourceRecordSetsInput, optFns ...func(*route53.Options)) (*route53.ListResourceRecordSetsOutput, error) |
| 83 | ChangeResourceRecordSets(ctx context.Context, in *route53.ChangeResourceRecordSetsInput, optFns ...func(*route53.Options)) (*route53.ChangeResourceRecordSetsOutput, error) |
| 84 | ListHealthChecks(ctx context.Context, in *route53.ListHealthChecksInput, optFns ...func(*route53.Options)) (*route53.ListHealthChecksOutput, error) |
| 85 | ListTagsForResources(ctx context.Context, in *route53.ListTagsForResourcesInput, optFns ...func(*route53.Options)) (*route53.ListTagsForResourcesOutput, error) |
| 86 | CreateHealthCheck(ctx context.Context, in *route53.CreateHealthCheckInput, optFns ...func(*route53.Options)) (*route53.CreateHealthCheckOutput, error) |
| 87 | DeleteHealthCheck(ctx context.Context, in *route53.DeleteHealthCheckInput, optFns ...func(*route53.Options)) (*route53.DeleteHealthCheckOutput, error) |
| 88 | ChangeTagsForResource(ctx context.Context, in *route53.ChangeTagsForResourceInput, optFns ...func(*route53.Options)) (*route53.ChangeTagsForResourceOutput, error) |
| 89 | } |
| 90 | |
| 91 | // route53Methods satisfies route53API from method values of one route53 client. |
| 92 | // Storing the *route53.Client itself in the interface would make the linker keep every exported Route53 method once html/template is linked in (discussion #275); method values keep only the calls made here. |
| 93 | type route53Methods struct { |
| 94 | listResourceRecordSets func(ctx context.Context, in *route53.ListResourceRecordSetsInput, optFns ...func(*route53.Options)) (*route53.ListResourceRecordSetsOutput, error) |
| 95 | changeResourceRecordSets func(ctx context.Context, in *route53.ChangeResourceRecordSetsInput, optFns ...func(*route53.Options)) (*route53.ChangeResourceRecordSetsOutput, error) |
| 96 | listHealthChecks func(ctx context.Context, in *route53.ListHealthChecksInput, optFns ...func(*route53.Options)) (*route53.ListHealthChecksOutput, error) |
| 97 | listTagsForResources func(ctx context.Context, in *route53.ListTagsForResourcesInput, optFns ...func(*route53.Options)) (*route53.ListTagsForResourcesOutput, error) |
| 98 | createHealthCheck func(ctx context.Context, in *route53.CreateHealthCheckInput, optFns ...func(*route53.Options)) (*route53.CreateHealthCheckOutput, error) |
| 99 | deleteHealthCheck func(ctx context.Context, in *route53.DeleteHealthCheckInput, optFns ...func(*route53.Options)) (*route53.DeleteHealthCheckOutput, error) |
| 100 | changeTagsForResource func(ctx context.Context, in *route53.ChangeTagsForResourceInput, optFns ...func(*route53.Options)) (*route53.ChangeTagsForResourceOutput, error) |
| 101 | } |
| 102 | |
| 103 | func (m route53Methods) ListResourceRecordSets(ctx context.Context, in *route53.ListResourceRecordSetsInput, opts ...func(*route53.Options)) (*route53.ListResourceRecordSetsOutput, error) { |
| 104 | return m.listResourceRecordSets(ctx, in, opts...) |
| 105 | } |
| 106 | |
| 107 | func (m route53Methods) ChangeResourceRecordSets(ctx context.Context, in *route53.ChangeResourceRecordSetsInput, opts ...func(*route53.Options)) (*route53.ChangeResourceRecordSetsOutput, error) { |
| 108 | return m.changeResourceRecordSets(ctx, in, opts...) |
| 109 | } |
| 110 | |
| 111 | func (m route53Methods) ListHealthChecks(ctx context.Context, in *route53.ListHealthChecksInput, opts ...func(*route53.Options)) (*route53.ListHealthChecksOutput, error) { |
| 112 | return m.listHealthChecks(ctx, in, opts...) |
| 113 | } |
| 114 | |
| 115 | func (m route53Methods) ListTagsForResources(ctx context.Context, in *route53.ListTagsForResourcesInput, opts ...func(*route53.Options)) (*route53.ListTagsForResourcesOutput, error) { |
| 116 | return m.listTagsForResources(ctx, in, opts...) |
| 117 | } |
| 118 | |
| 119 | func (m route53Methods) CreateHealthCheck(ctx context.Context, in *route53.CreateHealthCheckInput, opts ...func(*route53.Options)) (*route53.CreateHealthCheckOutput, error) { |
| 120 | return m.createHealthCheck(ctx, in, opts...) |
| 121 | } |
| 122 | |
| 123 | func (m route53Methods) DeleteHealthCheck(ctx context.Context, in *route53.DeleteHealthCheckInput, opts ...func(*route53.Options)) (*route53.DeleteHealthCheckOutput, error) { |
| 124 | return m.deleteHealthCheck(ctx, in, opts...) |
| 125 | } |
| 126 | |
| 127 | func (m route53Methods) ChangeTagsForResource(ctx context.Context, in *route53.ChangeTagsForResourceInput, opts ...func(*route53.Options)) (*route53.ChangeTagsForResourceOutput, error) { |
| 128 | return m.changeTagsForResource(ctx, in, opts...) |
| 129 | } |
| 130 | |
| 131 | func newRoute53Methods(c *route53.Client) route53Methods { |
| 132 | return route53Methods{ |
| 133 | listResourceRecordSets: c.ListResourceRecordSets, |
| 134 | changeResourceRecordSets: c.ChangeResourceRecordSets, |
| 135 | listHealthChecks: c.ListHealthChecks, |
| 136 | listTagsForResources: c.ListTagsForResources, |
| 137 | createHealthCheck: c.CreateHealthCheck, |
| 138 | deleteHealthCheck: c.DeleteHealthCheck, |
| 139 | changeTagsForResource: c.ChangeTagsForResource, |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | // SteeringZoneID returns the deployment's validated steering hosted-zone id, |
| 144 | // or "" when the deployment is dormant (env unset, blank, or malformed — |
| 145 | // malformed is logged, because a typo'd zone id is a configuration error |
| 146 | // somebody needs to see, not a quiet no-op). |
| 147 | func SteeringZoneID() string { |
| 148 | raw := strings.TrimSpace(os.Getenv(steeringZoneEnv)) |
| 149 | if raw == "" { |
| 150 | return "" |
| 151 | } |
| 152 | if !steeringZonePattern.MatchString(raw) { |
| 153 | log.Printf("steering: %s=%q is not a bare hosted-zone id (want e.g. Z0ABCDEF1234) — staying dormant", steeringZoneEnv, raw) |
| 154 | return "" |
| 155 | } |
| 156 | return raw |
| 157 | } |
| 158 | |
| 159 | // NewClient builds a Route53 client from the default AWS credential chain — |
| 160 | // the box instance profile in production, exactly like |
| 161 | // internal/edge/route53dns.go's newRoute53DNS. Route53 is a global service |
| 162 | // but the SDK still needs a signing region, so default to us-east-1 when the |
| 163 | // environment names none; an explicit AWS_REGION still wins. |
| 164 | func NewClient(ctx context.Context) (route53API, error) { |
| 165 | var opts []func(*awsconfig.LoadOptions) error |
| 166 | if strings.TrimSpace(os.Getenv("AWS_REGION")) == "" && strings.TrimSpace(os.Getenv("AWS_DEFAULT_REGION")) == "" { |
| 167 | opts = append(opts, awsconfig.WithRegion("us-east-1")) |
| 168 | } |
| 169 | cfg, err := rootaws.LoadDefaultConfig(ctx, opts...) |
| 170 | if err != nil { |
| 171 | return nil, fmt.Errorf("steering: load aws config: %w", err) |
| 172 | } |
| 173 | return newRoute53Methods(route53.NewFromConfig(cfg)), nil |
| 174 | } |
| 175 | |
| 176 | // ownedCheck is one Route53 health check this deployment owns: tagged |
| 177 | // carlos-steering=<zoneID>, and therefore the only kind this code will ever |
| 178 | // delete. |
| 179 | type ownedCheck struct { |
| 180 | id string |
| 181 | eip string |
| 182 | } |
| 183 | |
| 184 | // converge makes the zone's carlos-managed latency records match want. |
| 185 | // |
| 186 | // The order is deliberate and is a safety property, not an implementation |
| 187 | // detail: |
| 188 | // |
| 189 | // 1. read ownership (which health checks are OURS) — a failure here aborts |
| 190 | // before anything is written, and an empty result means the managed |
| 191 | // record set is empty, which means NOTHING is deletable; |
| 192 | // 2. read the zone; |
| 193 | // 3. drop names Route53 will not let us write (plainConflicts) and decide |
| 194 | // what the pass would delete — BOTH REFUSALS RUN BEFORE THE FIRST WRITE, |
| 195 | // so a refused pass is genuinely zero-write, not merely zero-delete; |
| 196 | // 4. CREATE any missing health check — capacity before records need it; |
| 197 | // 5. UPSERT wanted records, DELETE managed-but-unwanted ones; |
| 198 | // 6. GC health checks nothing references any more — after the records that |
| 199 | // referenced them are gone, never before. |
| 200 | // |
| 201 | // A failure at any step returns loudly and leaves the rest of the pass |
| 202 | // undone; the housekeeper's next tick re-runs the whole thing from a fresh |
| 203 | // read. Every step is individually idempotent, so a half-applied pass is a |
| 204 | // state this function converges from, not a state it gets stuck in. |
| 205 | func converge(ctx context.Context, client route53API, zoneID string, want []desiredRecord) error { |
| 206 | if !steeringZonePattern.MatchString(strings.TrimSpace(zoneID)) { |
| 207 | return fmt.Errorf("steering: refusing to converge: %q is not a bare hosted-zone id", zoneID) |
| 208 | } |
| 209 | zoneID = strings.TrimSpace(zoneID) |
| 210 | |
| 211 | // An EMPTY desired set is never an instruction to tear down production |
| 212 | // DNS. desiredState only returns nothing when zero edges are armed (or |
| 213 | // there are no steered hosts AND no apps domain), and the overwhelmingly |
| 214 | // likelier cause of that on a live deployment is a degraded read — a |
| 215 | // control-bucket blip, a listing that came back short — not an operator |
| 216 | // deciding every edge should stop answering. Deleting the whole managed |
| 217 | // set on that signal would turn a transient read failure into a global |
| 218 | // outage. So: refuse, loudly, and let the next tick decide again. |
| 219 | // A genuine full teardown is an operator action. |
| 220 | // |
| 221 | // This is the degenerate case of the proportional guard below; both |
| 222 | // exist because "delete everything" and "delete most things" are the |
| 223 | // same mistake at different sizes. |
| 224 | if len(want) == 0 { |
| 225 | log.Printf("steering: empty desired set for zone %s — refusing to remove any record (a wholesale teardown is an operator action, not a converge outcome)", zoneID) |
| 226 | return nil |
| 227 | } |
| 228 | |
| 229 | owned, err := ownedHealthChecks(ctx, client, zoneID) |
| 230 | if err != nil { |
| 231 | return err |
| 232 | } |
| 233 | ownedIDs := make(map[string]bool, len(owned)) |
| 234 | checkByEIP := make(map[string]string, len(owned)) |
| 235 | for _, c := range owned { |
| 236 | ownedIDs[c.id] = true |
| 237 | // Deterministic winner when a leak has left two checks on one EIP: |
| 238 | // lowest id. The loser is unreferenced after this pass and the GC |
| 239 | // below collects it — the leak self-heals. |
| 240 | if have, ok := checkByEIP[c.eip]; !ok || c.id < have { |
| 241 | checkByEIP[c.eip] = c.id |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | existing, err := listRecords(ctx, client, zoneID) |
| 246 | if err != nil { |
| 247 | return err |
| 248 | } |
| 249 | |
| 250 | // 3a. names Route53 forbids us from writing at all. |
| 251 | conflicted := conflictedNames(want, existing) |
| 252 | if len(conflicted) > 0 { |
| 253 | want = dropNames(want, conflicted) |
| 254 | if len(want) == 0 { |
| 255 | log.Printf("steering: every steered name in zone %s collides with an existing record — refusing to change anything", zoneID) |
| 256 | return nil |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | // 3b. the proportional shrink guard. |
| 261 | managed, unwanted := partitionManaged(want, existing, ownedIDs, conflicted) |
| 262 | if limit := shrinkLimit(managed); len(unwanted) > limit { |
| 263 | for _, rr := range unwanted { |
| 264 | log.Printf("steering: would delete %s (set %s, value %s)", aws.ToString(rr.Name), aws.ToString(rr.SetIdentifier), firstValue(rr)) |
| 265 | } |
| 266 | log.Printf("steering: refusing to converge zone %s: the pass would delete %d of %d managed records (limit %d) — a shrink that large is far likelier a short read of edges or routes than an intent to withdraw those names, and a genuine teardown that size is an operator action, exactly like the empty-desired-set case; the intended deletions are listed above", |
| 267 | zoneID, len(unwanted), managed, limit) |
| 268 | return nil |
| 269 | } |
| 270 | |
| 271 | // 4. health checks for every wanted EIP. |
| 272 | sni := steeringHostname(want) |
| 273 | loggedSNIFallback := false |
| 274 | for _, eip := range distinctEIPs(want) { |
| 275 | if _, ok := checkByEIP[eip]; ok { |
| 276 | continue |
| 277 | } |
| 278 | fqdnFor := sni |
| 279 | if fqdnFor == "" { |
| 280 | fqdnFor = fallbackHostname(want, eip) |
| 281 | // The shared steering hostname is the RIGHT SNI name because |
| 282 | // every edge holds a certificate for it — a check on it tests |
| 283 | // the path every visitor takes. Falling back to one member's |
| 284 | // hostname is a real downgrade: the check now depends on that |
| 285 | // one member's name and certificate, so if the member later |
| 286 | // moves or its route is removed, the check starts failing for |
| 287 | // reasons that have nothing to do with the edge's health. |
| 288 | // |
| 289 | // Note the interaction worth seeing in a log: if the synthetic |
| 290 | // edge.<apps-domain> name was DROPPED for a record conflict just |
| 291 | // above, this is where that shows up — a name-level conflict |
| 292 | // silently degrading every health check created afterwards. |
| 293 | if !loggedSNIFallback { |
| 294 | log.Printf("steering: no edge.<apps-domain> record in the desired set — new health checks fall back to a member hostname (%s) for SNI instead of the shared steering hostname; if that name was skipped for a record conflict above, THAT is the cause, and the check now rides one member's name and certificate", fqdnFor) |
| 295 | loggedSNIFallback = true |
| 296 | } |
| 297 | } |
| 298 | id, err := createHealthCheck(ctx, client, zoneID, eip, fqdnFor) |
| 299 | if err != nil { |
| 300 | return err |
| 301 | } |
| 302 | checkByEIP[eip] = id |
| 303 | ownedIDs[id] = true |
| 304 | owned = append(owned, ownedCheck{id: id, eip: eip}) |
| 305 | } |
| 306 | |
| 307 | // 5. records. |
| 308 | changes, err := recordChanges(want, existing, checkByEIP, ownedIDs, unwanted) |
| 309 | if err != nil { |
| 310 | return err |
| 311 | } |
| 312 | if err := applyChanges(ctx, client, zoneID, changes); err != nil { |
| 313 | return err |
| 314 | } |
| 315 | |
| 316 | // 6. GC. Runs only once the record batches have landed. |
| 317 | return gcHealthChecks(ctx, client, owned, want, existing, changes) |
| 318 | } |
| 319 | |
| 320 | // shrinkLimit is how many managed records one pass may delete, given how many |
| 321 | // it currently manages: half of them, but never fewer than shrinkFloor. |
| 322 | // |
| 323 | // FINDING 1 (review, 2026-08-22). The empty-desired-set refusal above only |
| 324 | // catches a read that returned NOTHING. A SHORT read — three of five edges' |
| 325 | // files listed, or a routes query that came back missing a steered host — has |
| 326 | // a non-empty want and sails straight past it, and the converger would |
| 327 | // dutifully delete every record belonging to the members that went missing. |
| 328 | // For a live steered host that is NXDOMAIN until the next tick: not a slow |
| 329 | // failover, a hard outage, caused by a transient read rather than by anything |
| 330 | // an operator did. |
| 331 | // |
| 332 | // The durable fix is fail-closed reads upstream (Task 6's half: a partial |
| 333 | // listing must produce an ERROR, not a smaller slice). This is the half that |
| 334 | // belongs here — a blast-radius cap that holds even if a caller one day hands |
| 335 | // us a quietly-truncated set. A pass that would remove most of what it manages |
| 336 | // stops and describes itself instead. |
| 337 | // |
| 338 | // The floor keeps small zones workable: a 2-record zone losing one record is |
| 339 | // ordinary, and a purely proportional rule would refuse it forever. The floor |
| 340 | // is small enough that the cap still bites at any size worth protecting. |
| 341 | func shrinkLimit(managed int) int { |
| 342 | if half := managed / 2; half > shrinkFloor { |
| 343 | return half |
| 344 | } |
| 345 | return shrinkFloor |
| 346 | } |
| 347 | |
| 348 | // --------------------------------------------------------------------------- |
| 349 | // ownership |
| 350 | // --------------------------------------------------------------------------- |
| 351 | |
| 352 | // ownedHealthChecks returns the account's health checks that THIS deployment |
| 353 | // owns, i.e. that carry carlos-steering=<zoneID>. |
| 354 | // |
| 355 | // Two narrowings, both in the safe direction: |
| 356 | // |
| 357 | // - the shape prefilter (HTTPS on 443 with an IPAddress, and not owned by |
| 358 | // another AWS service) only ever SHRINKS the set of checks that can be |
| 359 | // deleted. Every check this code creates has exactly that shape, so a |
| 360 | // tagged check of some other shape is one we did not create — ignoring it |
| 361 | // costs a stranded check; deleting it could cost someone else's |
| 362 | // monitoring. |
| 363 | // - the tag is then checked for real, per candidate. Shape alone never |
| 364 | // confers ownership: the tag is the authority. |
| 365 | // |
| 366 | // A listing or tag call that errors aborts the whole pass. That matters: if |
| 367 | // this function returned a partial set, checks it failed to see would look |
| 368 | // unowned (so records bound to them would look unmanaged — harmless) but, |
| 369 | // worse, their EIPs would look uncovered and a duplicate check would be |
| 370 | // created. Failing loudly is the only correct response. |
| 371 | func ownedHealthChecks(ctx context.Context, client route53API, zoneID string) ([]ownedCheck, error) { |
| 372 | type candidate struct{ id, eip string } |
| 373 | var candidates []candidate |
| 374 | |
| 375 | var marker *string |
| 376 | for { |
| 377 | out, err := client.ListHealthChecks(ctx, &route53.ListHealthChecksInput{Marker: marker}) |
| 378 | if err != nil { |
| 379 | return nil, fmt.Errorf("steering: list health checks: %w", err) |
| 380 | } |
| 381 | for _, hc := range out.HealthChecks { |
| 382 | cfg := hc.HealthCheckConfig |
| 383 | if cfg == nil || hc.LinkedService != nil { |
| 384 | continue |
| 385 | } |
| 386 | if cfg.Type != r53types.HealthCheckTypeHttps || aws.ToInt32(cfg.Port) != healthCheckPort { |
| 387 | continue |
| 388 | } |
| 389 | eip := aws.ToString(cfg.IPAddress) |
| 390 | if eip == "" { |
| 391 | continue |
| 392 | } |
| 393 | candidates = append(candidates, candidate{id: aws.ToString(hc.Id), eip: eip}) |
| 394 | } |
| 395 | if !out.IsTruncated || aws.ToString(out.NextMarker) == "" { |
| 396 | break |
| 397 | } |
| 398 | marker = out.NextMarker |
| 399 | } |
| 400 | if len(candidates) == 0 { |
| 401 | return nil, nil |
| 402 | } |
| 403 | sort.Slice(candidates, func(i, j int) bool { return candidates[i].id < candidates[j].id }) |
| 404 | |
| 405 | eipByID := make(map[string]string, len(candidates)) |
| 406 | ids := make([]string, 0, len(candidates)) |
| 407 | for _, c := range candidates { |
| 408 | eipByID[c.id] = c.eip |
| 409 | ids = append(ids, c.id) |
| 410 | } |
| 411 | |
| 412 | var owned []ownedCheck |
| 413 | for start := 0; start < len(ids); start += healthCheckTagBatch { |
| 414 | end := start + healthCheckTagBatch |
| 415 | if end > len(ids) { |
| 416 | end = len(ids) |
| 417 | } |
| 418 | out, err := client.ListTagsForResources(ctx, &route53.ListTagsForResourcesInput{ |
| 419 | ResourceType: r53types.TagResourceTypeHealthcheck, |
| 420 | ResourceIds: ids[start:end], |
| 421 | }) |
| 422 | if err != nil { |
| 423 | return nil, fmt.Errorf("steering: list health-check tags: %w", err) |
| 424 | } |
| 425 | for _, set := range out.ResourceTagSets { |
| 426 | id := aws.ToString(set.ResourceId) |
| 427 | for _, tg := range set.Tags { |
| 428 | if aws.ToString(tg.Key) == steeringTagKey && aws.ToString(tg.Value) == zoneID { |
| 429 | owned = append(owned, ownedCheck{id: id, eip: eipByID[id]}) |
| 430 | break |
| 431 | } |
| 432 | } |
| 433 | } |
| 434 | } |
| 435 | sort.Slice(owned, func(i, j int) bool { return owned[i].id < owned[j].id }) |
| 436 | return owned, nil |
| 437 | } |
| 438 | |
| 439 | // createHealthCheck mints one check for eip and tags it for this zone. |
| 440 | // |
| 441 | // Create-then-tag is two calls and Route53 offers no atomic form. An |
| 442 | // UNTAGGED check is the worst outcome available: this code would never reuse |
| 443 | // it (untagged is not ours) and never GC it (the GC is tag-scoped), so every |
| 444 | // subsequent pass would mint another one — an unbounded leak at the |
| 445 | // housekeeper's cadence. So a tagging failure best-effort deletes the check |
| 446 | // it just made before failing the pass. If that delete ALSO fails, the check |
| 447 | // is named in the log so an operator can find it. |
| 448 | func createHealthCheck(ctx context.Context, client route53API, zoneID, eip, fqdn string) (string, error) { |
| 449 | // CallerReference must be unique per creation: Route53 remembers it for |
| 450 | // deleted checks for a while, so a name derived only from the EIP would |
| 451 | // make a delete-then-recreate (edge disarms, then rearms) fail with |
| 452 | // HealthCheckAlreadyExists. The timestamp keeps it under the 64-char cap |
| 453 | // while staying unique. Ownership is NEVER read from this string — the |
| 454 | // tag is the authority — so its only job is uniqueness plus a human hint |
| 455 | // in the console. |
| 456 | ref := fmt.Sprintf("%s%s-%d", healthCheckCallerRefPrefix, eip, time.Now().UnixNano()) |
| 457 | out, err := client.CreateHealthCheck(ctx, &route53.CreateHealthCheckInput{ |
| 458 | CallerReference: aws.String(ref), |
| 459 | HealthCheckConfig: &r53types.HealthCheckConfig{ |
| 460 | Type: r53types.HealthCheckTypeHttps, |
| 461 | Port: aws.Int32(healthCheckPort), |
| 462 | IPAddress: aws.String(eip), |
| 463 | FullyQualifiedDomainName: aws.String(fqdn), |
| 464 | ResourcePath: aws.String(healthCheckPath), |
| 465 | EnableSNI: aws.Bool(true), |
| 466 | RequestInterval: aws.Int32(healthCheckInterval), |
| 467 | FailureThreshold: aws.Int32(healthCheckFailureThresh), |
| 468 | MeasureLatency: aws.Bool(false), |
| 469 | }, |
| 470 | }) |
| 471 | if err != nil { |
| 472 | return "", fmt.Errorf("steering: create health check for %s: %w", eip, err) |
| 473 | } |
| 474 | id := "" |
| 475 | if out.HealthCheck != nil { |
| 476 | id = aws.ToString(out.HealthCheck.Id) |
| 477 | } |
| 478 | if id == "" { |
| 479 | return "", fmt.Errorf("steering: create health check for %s: no id returned", eip) |
| 480 | } |
| 481 | |
| 482 | if _, err := client.ChangeTagsForResource(ctx, &route53.ChangeTagsForResourceInput{ |
| 483 | ResourceType: r53types.TagResourceTypeHealthcheck, |
| 484 | ResourceId: aws.String(id), |
| 485 | AddTags: []r53types.Tag{{Key: aws.String(steeringTagKey), Value: aws.String(zoneID)}}, |
| 486 | }); err != nil { |
| 487 | if _, delErr := client.DeleteHealthCheck(ctx, &route53.DeleteHealthCheckInput{HealthCheckId: aws.String(id)}); delErr != nil { |
| 488 | log.Printf("steering: LEAKED health check %s for %s: tagging failed (%v) and so did the cleanup delete (%v) — delete it by hand", id, eip, err, delErr) |
| 489 | } |
| 490 | return "", fmt.Errorf("steering: tag health check %s for %s: %w", id, eip, err) |
| 491 | } |
| 492 | log.Printf("steering: created health check %s for %s (HTTPS:%d, SNI %s), tagged %s=%s", id, eip, healthCheckPort, fqdn, steeringTagKey, zoneID) |
| 493 | return id, nil |
| 494 | } |
| 495 | |
| 496 | // gcHealthChecks deletes owned checks that nothing references any more. |
| 497 | // |
| 498 | // Two independent gates, both of which must pass: |
| 499 | // |
| 500 | // - the check is OURS (it is in owned, which is tag-derived — see |
| 501 | // ownedHealthChecks). This is the gate that makes it structurally |
| 502 | // impossible to delete a check belonging to another deployment, another |
| 503 | // tool, or a person. |
| 504 | // - no record SURVIVING this pass references it: not one of the records we |
| 505 | // just wrote, and not one of the zone's records we left alone (which |
| 506 | // includes every static and foreign record). Route53 would refuse a |
| 507 | // delete of an in-use check anyway, but failing the whole pass on that |
| 508 | // is worse than skipping the one check and saying so. |
| 509 | // |
| 510 | // A delete that fails is logged, not fatal: the records are already correct, |
| 511 | // and next tick will try again. |
| 512 | func gcHealthChecks(ctx context.Context, client route53API, owned []ownedCheck, want []desiredRecord, existing []r53types.ResourceRecordSet, applied []r53types.Change) error { |
| 513 | if len(owned) == 0 { |
| 514 | return nil |
| 515 | } |
| 516 | |
| 517 | wantedEIP := make(map[string]bool, len(want)) |
| 518 | for _, rec := range want { |
| 519 | wantedEIP[rec.EIP] = true |
| 520 | } |
| 521 | |
| 522 | // Every check id still referenced once this pass has landed. |
| 523 | deleted := make(map[string]bool) |
| 524 | for _, c := range applied { |
| 525 | if c.Action == r53types.ChangeActionDelete && c.ResourceRecordSet != nil { |
| 526 | deleted[recordKey(*c.ResourceRecordSet)] = true |
| 527 | } |
| 528 | } |
| 529 | referenced := make(map[string]bool) |
| 530 | for _, rr := range existing { |
| 531 | if deleted[recordKey(rr)] { |
| 532 | continue |
| 533 | } |
| 534 | if id := aws.ToString(rr.HealthCheckId); id != "" { |
| 535 | referenced[id] = true |
| 536 | } |
| 537 | } |
| 538 | for _, c := range applied { |
| 539 | if c.Action == r53types.ChangeActionUpsert && c.ResourceRecordSet != nil { |
| 540 | if id := aws.ToString(c.ResourceRecordSet.HealthCheckId); id != "" { |
| 541 | referenced[id] = true |
| 542 | } |
| 543 | } |
| 544 | } |
| 545 | |
| 546 | for _, c := range owned { |
| 547 | if wantedEIP[c.eip] && referenced[c.id] { |
| 548 | continue |
| 549 | } |
| 550 | if referenced[c.id] { |
| 551 | log.Printf("steering: health check %s (%s) is no longer wanted but a surviving record still references it — leaving it in place", c.id, c.eip) |
| 552 | continue |
| 553 | } |
| 554 | if _, err := client.DeleteHealthCheck(ctx, &route53.DeleteHealthCheckInput{HealthCheckId: aws.String(c.id)}); err != nil { |
| 555 | log.Printf("steering: could not delete unreferenced health check %s (%s): %v — retrying next pass", c.id, c.eip, err) |
| 556 | continue |
| 557 | } |
| 558 | log.Printf("steering: deleted unreferenced health check %s (%s)", c.id, c.eip) |
| 559 | } |
| 560 | return nil |
| 561 | } |
| 562 | |
| 563 | // --------------------------------------------------------------------------- |
| 564 | // records |
| 565 | // --------------------------------------------------------------------------- |
| 566 | |
| 567 | // listRecords pages the whole zone. Every page matters: a converger that |
| 568 | // stopped early would see records it manages as absent and rewrite them, and |
| 569 | // would see managed-but-unwanted ones as already gone. |
| 570 | func listRecords(ctx context.Context, client route53API, zoneID string) ([]r53types.ResourceRecordSet, error) { |
| 571 | var out []r53types.ResourceRecordSet |
| 572 | in := &route53.ListResourceRecordSetsInput{HostedZoneId: aws.String(zoneID)} |
| 573 | for { |
| 574 | page, err := client.ListResourceRecordSets(ctx, in) |
| 575 | if err != nil { |
| 576 | return nil, fmt.Errorf("steering: list record sets in %s: %w", zoneID, err) |
| 577 | } |
| 578 | out = append(out, page.ResourceRecordSets...) |
| 579 | if !page.IsTruncated || aws.ToString(page.NextRecordName) == "" { |
| 580 | return out, nil |
| 581 | } |
| 582 | in = &route53.ListResourceRecordSetsInput{ |
| 583 | HostedZoneId: aws.String(zoneID), |
| 584 | StartRecordName: page.NextRecordName, |
| 585 | StartRecordType: page.NextRecordType, |
| 586 | StartRecordIdentifier: page.NextRecordIdentifier, |
| 587 | } |
| 588 | } |
| 589 | } |
| 590 | |
| 591 | // recordKey is a record set's identity: Route53 keys a resource record set on |
| 592 | // (name, type, set identifier), and so does this converger. Names are |
| 593 | // lowercased because DNS is case-insensitive and desiredRecord.Name is |
| 594 | // already lowercase by contract (steering.go's fqdn) — the listing side is |
| 595 | // the only place a stray case could enter. |
| 596 | func recordKey(rr r53types.ResourceRecordSet) string { |
| 597 | return strings.ToLower(aws.ToString(rr.Name)) + "|" + string(rr.Type) + "|" + aws.ToString(rr.SetIdentifier) |
| 598 | } |
| 599 | |
| 600 | func wantKey(rec desiredRecord) string { |
| 601 | return rec.Name + "|" + string(r53types.RRTypeA) + "|" + rec.Box |
| 602 | } |
| 603 | |
| 604 | // isManaged is THE deletion boundary. A record in the steering zone is |
| 605 | // carlos-managed — and therefore something this code may delete — only if ALL |
| 606 | // of the following hold: |
| 607 | // |
| 608 | // - it is an A record. Nothing else is ever written here. |
| 609 | // - it carries a SetIdentifier. A record WITHOUT one is a plain, |
| 610 | // single-answer record: the zone's apex, its www, its MX/NS/SOA, an |
| 611 | // operator's hand-made host. Those are STATIC / OPERATOR TERRITORY and |
| 612 | // are never listed as deletable, full stop. This is the rule the brief |
| 613 | // names and the one that must never be relaxed. |
| 614 | // - it is a LATENCY record (Region set). A weighted, failover, geo or |
| 615 | // multivalue record also carries a SetIdentifier and is somebody else's |
| 616 | // routing policy. |
| 617 | // - it is not an alias. This converger writes plain A records to EIPs; |
| 618 | // an alias record under our key was made by something else. |
| 619 | // - it is bound to a health check WE own (tag-derived, ownedHealthChecks). |
| 620 | // This is the strongest of the five: every record this code writes is |
| 621 | // bound to one of our tagged checks, so the binding is a creation |
| 622 | // signature. It is what makes the answer to "can converge delete a |
| 623 | // record it did not create?" a structural no rather than a naming |
| 624 | // convention. |
| 625 | // |
| 626 | // The last gate has a cost, taken deliberately: a managed record whose health |
| 627 | // check was deleted out from under us becomes invisible to this code and will |
| 628 | // linger. It is logged by name every pass (see recordChanges) so it is loud |
| 629 | // rather than silent, and a stale record is a far cheaper failure than a |
| 630 | // wrongly-deleted one. |
| 631 | func isManaged(rr r53types.ResourceRecordSet, ownedIDs map[string]bool) bool { |
| 632 | if rr.Type != r53types.RRTypeA { |
| 633 | return false |
| 634 | } |
| 635 | if aws.ToString(rr.SetIdentifier) == "" { |
| 636 | return false |
| 637 | } |
| 638 | if rr.Region == "" { |
| 639 | return false |
| 640 | } |
| 641 | if rr.AliasTarget != nil { |
| 642 | return false |
| 643 | } |
| 644 | return ownedIDs[aws.ToString(rr.HealthCheckId)] |
| 645 | } |
| 646 | |
| 647 | // desiredRRSet renders one desiredRecord as the record set Route53 should |
| 648 | // hold. TTL 60, type A, plain value (never an AliasTarget), latency routing |
| 649 | // keyed on the box label, bound to the box's EIP health check. |
| 650 | func desiredRRSet(rec desiredRecord, checkID string) r53types.ResourceRecordSet { |
| 651 | return r53types.ResourceRecordSet{ |
| 652 | Name: aws.String(rec.Name), |
| 653 | Type: r53types.RRTypeA, |
| 654 | SetIdentifier: aws.String(rec.Box), |
| 655 | Region: r53types.ResourceRecordSetRegion(rec.Region), |
| 656 | TTL: aws.Int64(recordTTL), |
| 657 | ResourceRecords: []r53types.ResourceRecord{{Value: aws.String(rec.EIP)}}, |
| 658 | HealthCheckId: aws.String(checkID), |
| 659 | } |
| 660 | } |
| 661 | |
| 662 | // sameRRSet reports whether the zone already holds exactly what we want. It |
| 663 | // is what makes a steady-state pass write NOTHING: every field this converger |
| 664 | // sets is compared, and every field it never sets is required to be absent |
| 665 | // (an unexpected Weight or GeoLocation on our key means somebody changed the |
| 666 | // routing policy underneath us, and we should rewrite it). |
| 667 | func sameRRSet(have, want r53types.ResourceRecordSet) bool { |
| 668 | if !strings.EqualFold(aws.ToString(have.Name), aws.ToString(want.Name)) { |
| 669 | return false |
| 670 | } |
| 671 | if have.Type != want.Type || aws.ToString(have.SetIdentifier) != aws.ToString(want.SetIdentifier) { |
| 672 | return false |
| 673 | } |
| 674 | if have.Region != want.Region || aws.ToInt64(have.TTL) != aws.ToInt64(want.TTL) { |
| 675 | return false |
| 676 | } |
| 677 | if aws.ToString(have.HealthCheckId) != aws.ToString(want.HealthCheckId) { |
| 678 | return false |
| 679 | } |
| 680 | if have.AliasTarget != nil || have.Weight != nil || have.GeoLocation != nil || |
| 681 | have.GeoProximityLocation != nil || have.CidrRoutingConfig != nil || |
| 682 | have.MultiValueAnswer != nil || have.Failover != "" { |
| 683 | return false |
| 684 | } |
| 685 | if len(have.ResourceRecords) != len(want.ResourceRecords) { |
| 686 | return false |
| 687 | } |
| 688 | for i := range have.ResourceRecords { |
| 689 | if aws.ToString(have.ResourceRecords[i].Value) != aws.ToString(want.ResourceRecords[i].Value) { |
| 690 | return false |
| 691 | } |
| 692 | } |
| 693 | return true |
| 694 | } |
| 695 | |
| 696 | // conflictedNames returns the wanted names this converger cannot write a |
| 697 | // latency A record at, lowercased with the trailing dot, because an existing |
| 698 | // record there makes the write illegal. |
| 699 | // |
| 700 | // FINDING 2 (review round 1) + FINDING A (round 2, 2026-08-22). Three record |
| 701 | // shapes at a wanted name are all rejected by Route53, and change batches are |
| 702 | // ATOMIC — so the rejection takes the whole batch with it. A single such |
| 703 | // record would wedge EVERY steering record in the zone, on every tick, |
| 704 | // forever: the pass would fail, retry identically, and fail again, while |
| 705 | // unrelated names never got their records. conflictReason enumerates the |
| 706 | // three; they share one failure mode and therefore one cure. |
| 707 | // |
| 708 | // The cure is to skip the conflicting NAME entirely — no upsert, no delete — |
| 709 | // and say so loudly, naming what is in the way. Never the alternative of |
| 710 | // deleting the offending record to make room: it is an operator's record |
| 711 | // (a plain record has no SetIdentifier, which is static/operator territory |
| 712 | // under the same rule isManaged enforces; a CNAME or a weighted set is |
| 713 | // somebody's deliberate routing), and taking a name away from its owner |
| 714 | // because a member typed it into a steering field is not a call this code |
| 715 | // gets to make. |
| 716 | // |
| 717 | // Scope note: this is a NAME-level refusal, and deliberately so — all three |
| 718 | // shapes make the whole name unwritable. It is not the right shape for a |
| 719 | // KEY-level conflict such as an existing ALIAS latency record at exactly one |
| 720 | // (name, SetIdentifier) we want, which Route53 also refuses to convert in |
| 721 | // place via UPSERT; that one is left open on the report rather than guessed |
| 722 | // at here, because dropping the whole name for it would refuse to steer names |
| 723 | // that are in fact steerable. |
| 724 | func conflictedNames(want []desiredRecord, existing []r53types.ResourceRecordSet) map[string]bool { |
| 725 | wantNames := make(map[string]bool, len(want)) |
| 726 | for _, rec := range want { |
| 727 | wantNames[strings.ToLower(rec.Name)] = true |
| 728 | } |
| 729 | conflicted := map[string]bool{} |
| 730 | for _, rr := range existing { |
| 731 | name := strings.ToLower(aws.ToString(rr.Name)) |
| 732 | if !wantNames[name] || conflicted[name] { |
| 733 | continue |
| 734 | } |
| 735 | reason := conflictReason(rr) |
| 736 | if reason == "" { |
| 737 | continue |
| 738 | } |
| 739 | conflicted[name] = true |
| 740 | log.Printf("steering: SKIPPING %s entirely: %s — steering this name needs that record changed by its owner first; every other steered name in this zone is unaffected", |
| 741 | aws.ToString(rr.Name), reason) |
| 742 | } |
| 743 | return conflicted |
| 744 | } |
| 745 | |
| 746 | // conflictReason names why an existing record makes its name unwritable by |
| 747 | // this converger, or "" if it does not. Three shapes, all with the SAME |
| 748 | // failure mode — the write is rejected and takes the whole atomic batch with |
| 749 | // it — and therefore all with the same cure: skip the name. |
| 750 | func conflictReason(rr r53types.ResourceRecordSet) string { |
| 751 | setID := aws.ToString(rr.SetIdentifier) |
| 752 | switch { |
| 753 | // A CNAME is EXCLUSIVE in DNS: a name holding one may hold no other |
| 754 | // record type at all. A member steering a hostname that already CNAMEs |
| 755 | // somewhere in the steering zone is not exotic — it is the ordinary |
| 756 | // custom-domain shape the spec itself recommends (CNAME to |
| 757 | // edge.<apps-domain>), pointed at a name inside the same zone. |
| 758 | case rr.Type == r53types.RRTypeCname: |
| 759 | return fmt.Sprintf("the zone already holds a CNAME there (-> %s, TTL %d), and a name holding a CNAME may hold no other record type", |
| 760 | firstValue(rr), aws.ToInt64(rr.TTL)) |
| 761 | |
| 762 | // A plain (simple) A record and a routing-policy record set cannot share |
| 763 | // a name+type. |
| 764 | case rr.Type == r53types.RRTypeA && setID == "": |
| 765 | return fmt.Sprintf("the zone already holds a plain A record there (value %s, TTL %d), and Route53 forbids a name from holding both a simple record and a routing-policy record set", |
| 766 | firstValue(rr), aws.ToInt64(rr.TTL)) |
| 767 | |
| 768 | // One routing POLICY per name+type: a weighted, failover, geo, geo- |
| 769 | // proximity, CIDR or multivalue A set at this name cannot coexist with a |
| 770 | // latency set. Latency records are the one policy we write, and they are |
| 771 | // identified by Region being set. |
| 772 | case rr.Type == r53types.RRTypeA && setID != "" && rr.Region == "": |
| 773 | return fmt.Sprintf("the zone already holds a NON-LATENCY A record set there (set %q, %s), and Route53 allows only one routing policy per name and type", |
| 774 | setID, otherPolicyName(rr)) |
| 775 | } |
| 776 | return "" |
| 777 | } |
| 778 | |
| 779 | // otherPolicyName describes which non-latency routing policy a record set |
| 780 | // uses, for the conflict log — an operator reading it should not have to go |
| 781 | // look the record up to know what they are being told about. |
| 782 | func otherPolicyName(rr r53types.ResourceRecordSet) string { |
| 783 | switch { |
| 784 | case rr.Weight != nil: |
| 785 | return "weighted" |
| 786 | case rr.Failover != "": |
| 787 | return "failover" |
| 788 | case rr.GeoLocation != nil: |
| 789 | return "geolocation" |
| 790 | case rr.GeoProximityLocation != nil: |
| 791 | return "geoproximity" |
| 792 | case rr.CidrRoutingConfig != nil: |
| 793 | return "CIDR" |
| 794 | case aws.ToBool(rr.MultiValueAnswer): |
| 795 | return "multivalue-answer" |
| 796 | } |
| 797 | return "a routing policy this converger does not write" |
| 798 | } |
| 799 | |
| 800 | // dropNames removes every desired record whose Name is in the set. |
| 801 | func dropNames(want []desiredRecord, names map[string]bool) []desiredRecord { |
| 802 | out := make([]desiredRecord, 0, len(want)) |
| 803 | for _, rec := range want { |
| 804 | if names[strings.ToLower(rec.Name)] { |
| 805 | continue |
| 806 | } |
| 807 | out = append(out, rec) |
| 808 | } |
| 809 | return out |
| 810 | } |
| 811 | |
| 812 | // partitionManaged counts the zone's carlos-managed records and collects the |
| 813 | // ones no desired key wants — the pass's would-be deletions. It is separated |
| 814 | // from recordChanges so converge can weigh the deletions BEFORE it creates a |
| 815 | // single health check, which is what makes the shrink guard's refusal |
| 816 | // genuinely zero-write. |
| 817 | // |
| 818 | // Records at a conflicted name are excluded from both halves: the pass has |
| 819 | // already promised not to touch that name. That exclusion is a STRUCTURAL |
| 820 | // BACKSTOP and is deliberately untested — every conflict shape |
| 821 | // (conflictReason) is one Route53 forbids from coexisting with a latency |
| 822 | // record set at that name, so a MANAGED record at a conflicted name is a |
| 823 | // state the service cannot produce. It is kept, in the same spirit as |
| 824 | // dedupeRecords in steering.go, so that a misread listing fails by leaving |
| 825 | // records alone rather than by deleting our own. |
| 826 | func partitionManaged(want []desiredRecord, existing []r53types.ResourceRecordSet, ownedIDs, conflicted map[string]bool) (managed int, unwanted []r53types.ResourceRecordSet) { |
| 827 | wanted := make(map[string]bool, len(want)) |
| 828 | for _, rec := range want { |
| 829 | wanted[wantKey(rec)] = true |
| 830 | } |
| 831 | for _, rr := range existing { |
| 832 | if conflicted[strings.ToLower(aws.ToString(rr.Name))] { |
| 833 | continue |
| 834 | } |
| 835 | if !isManaged(rr, ownedIDs) { |
| 836 | // Say so when it LOOKS like ours but fails the ownership gate — |
| 837 | // a latency A record with a SetIdentifier whose health check we |
| 838 | // do not own is either an operator's or an orphan of ours whose |
| 839 | // check went missing. Either way we do not touch it, and either |
| 840 | // way somebody should know it is there. |
| 841 | // |
| 842 | // NOT when this pass WANTS that key, though: recordChanges is |
| 843 | // about to adopt it, so "leaving it alone" would be a lie about |
| 844 | // the one record here that does get taken over. (The wanted |
| 845 | // check was lost when this loop was split out of recordChanges |
| 846 | // in review round 1; round 2 caught the regression.) |
| 847 | if rr.Type == r53types.RRTypeA && aws.ToString(rr.SetIdentifier) != "" && rr.Region != "" && rr.AliasTarget == nil && |
| 848 | !wanted[recordKey(rr)] { |
| 849 | log.Printf("steering: leaving latency record %s (set %s) alone: health check %q is not one this deployment owns", |
| 850 | aws.ToString(rr.Name), aws.ToString(rr.SetIdentifier), aws.ToString(rr.HealthCheckId)) |
| 851 | } |
| 852 | continue |
| 853 | } |
| 854 | managed++ |
| 855 | if !wanted[recordKey(rr)] { |
| 856 | unwanted = append(unwanted, rr) |
| 857 | } |
| 858 | } |
| 859 | return managed, unwanted |
| 860 | } |
| 861 | |
| 862 | // firstValue renders a record set's first value for a log line. |
| 863 | func firstValue(rr r53types.ResourceRecordSet) string { |
| 864 | if len(rr.ResourceRecords) == 0 { |
| 865 | return "?" |
| 866 | } |
| 867 | return aws.ToString(rr.ResourceRecords[0].Value) |
| 868 | } |
| 869 | |
| 870 | // recordChanges diffs want against the zone. The deletions were decided |
| 871 | // already, by partitionManaged, and are passed in — this function only turns |
| 872 | // them into changes. |
| 873 | // |
| 874 | // UPSERTs come first in the returned slice and DELETEs last, so that when the |
| 875 | // changes are chunked, capacity is added before any is taken away. |
| 876 | // |
| 877 | // A wanted key that already holds a record we do NOT manage (a foreign |
| 878 | // latency record at exactly the (name, SetIdentifier) we would create) is |
| 879 | // ADOPTED by the UPSERT — CREATE would collide with it and there is no third |
| 880 | // option. It is logged, because taking over a record is the one write here |
| 881 | // that changes something we did not make. Note the difference from the |
| 882 | // plain-record conflict above: adoption replaces a record of the SAME routing |
| 883 | // policy at the SAME key, which Route53 permits and which is the record we |
| 884 | // would otherwise have created; the plain-record case is a different policy |
| 885 | // at the same name, which Route53 refuses outright. |
| 886 | func recordChanges(want []desiredRecord, existing []r53types.ResourceRecordSet, checkByEIP map[string]string, ownedIDs map[string]bool, unwanted []r53types.ResourceRecordSet) ([]r53types.Change, error) { |
| 887 | byKey := make(map[string]r53types.ResourceRecordSet, len(existing)) |
| 888 | for _, rr := range existing { |
| 889 | byKey[recordKey(rr)] = rr |
| 890 | } |
| 891 | |
| 892 | var upserts []r53types.Change |
| 893 | for _, rec := range want { |
| 894 | checkID, ok := checkByEIP[rec.EIP] |
| 895 | if !ok || checkID == "" { |
| 896 | // Unreachable: converge creates a check for every distinct EIP |
| 897 | // before calling this. Loud rather than silently minting a |
| 898 | // record with no health check, which would never withdraw. |
| 899 | return nil, fmt.Errorf("steering: no health check for %s (record %s / %s)", rec.EIP, rec.Name, rec.Box) |
| 900 | } |
| 901 | desired := desiredRRSet(rec, checkID) |
| 902 | have, present := byKey[wantKey(rec)] |
| 903 | if present && sameRRSet(have, desired) { |
| 904 | continue |
| 905 | } |
| 906 | if present && !isManaged(have, ownedIDs) { |
| 907 | log.Printf("steering: adopting existing record %s (set %s) — it sits at a name/identifier this deployment steers", rec.Name, rec.Box) |
| 908 | } |
| 909 | upserts = append(upserts, r53types.Change{Action: r53types.ChangeActionUpsert, ResourceRecordSet: &desired}) |
| 910 | } |
| 911 | |
| 912 | deletes := make([]r53types.Change, 0, len(unwanted)) |
| 913 | for _, rr := range unwanted { |
| 914 | set := rr // DELETE must resubmit the record set verbatim |
| 915 | deletes = append(deletes, r53types.Change{Action: r53types.ChangeActionDelete, ResourceRecordSet: &set}) |
| 916 | } |
| 917 | |
| 918 | return append(upserts, deletes...), nil |
| 919 | } |
| 920 | |
| 921 | // applyChanges submits the changes in chunks. |
| 922 | // |
| 923 | // Partial failure is explicit: chunk N lands, chunk N+1 fails, the pass |
| 924 | // returns the error and stops. That is safe because each chunk is |
| 925 | // independently valid — an UPSERT is idempotent and a DELETE is of a record |
| 926 | // nothing wants — so the next housekeeper tick re-reads the zone and issues |
| 927 | // exactly the changes still outstanding. The one thing that must NOT happen |
| 928 | // after a failed batch is the health-check GC, and converge returns before |
| 929 | // reaching it. |
| 930 | func applyChanges(ctx context.Context, client route53API, zoneID string, changes []r53types.Change) error { |
| 931 | if len(changes) == 0 { |
| 932 | return nil |
| 933 | } |
| 934 | for start := 0; start < len(changes); start += changeBatchLimit { |
| 935 | end := start + changeBatchLimit |
| 936 | if end > len(changes) { |
| 937 | end = len(changes) |
| 938 | } |
| 939 | chunk := changes[start:end] |
| 940 | if _, err := client.ChangeResourceRecordSets(ctx, &route53.ChangeResourceRecordSetsInput{ |
| 941 | HostedZoneId: aws.String(zoneID), |
| 942 | ChangeBatch: &r53types.ChangeBatch{Changes: chunk}, |
| 943 | }); err != nil { |
| 944 | return fmt.Errorf("steering: change batch %d-%d of %d in %s: %w", start, end, len(changes), zoneID, err) |
| 945 | } |
| 946 | for _, c := range chunk { |
| 947 | log.Printf("steering: %s %s (set %s)", c.Action, aws.ToString(c.ResourceRecordSet.Name), aws.ToString(c.ResourceRecordSet.SetIdentifier)) |
| 948 | } |
| 949 | } |
| 950 | return nil |
| 951 | } |
| 952 | |
| 953 | // --------------------------------------------------------------------------- |
| 954 | // naming |
| 955 | // --------------------------------------------------------------------------- |
| 956 | |
| 957 | // distinctEIPs returns want's EIPs, sorted, so health checks are created in a |
| 958 | // stable order. |
| 959 | func distinctEIPs(want []desiredRecord) []string { |
| 960 | seen := make(map[string]bool, len(want)) |
| 961 | out := make([]string, 0, len(want)) |
| 962 | for _, rec := range want { |
| 963 | if rec.EIP == "" || seen[rec.EIP] { |
| 964 | continue |
| 965 | } |
| 966 | seen[rec.EIP] = true |
| 967 | out = append(out, rec.EIP) |
| 968 | } |
| 969 | sort.Strings(out) |
| 970 | return out |
| 971 | } |
| 972 | |
| 973 | // steeringHostname finds the synthetic edge.<apps-domain> name in want and |
| 974 | // returns it dot-trimmed, for use as a health check's |
| 975 | // FullyQualifiedDomainName — an SNI/Host value, not a Route53 record name, so |
| 976 | // the trailing dot comes off. |
| 977 | // |
| 978 | // It is the right SNI name because every edge holds a certificate covering |
| 979 | // it, so the check exercises the same TLS path a real visitor takes rather |
| 980 | // than one host's certificate in particular. |
| 981 | // |
| 982 | // A member route literally NAMED edge.<apps-domain> produces the identical |
| 983 | // desired records, and desiredState dedupes on (Name, Box) — so this |
| 984 | // converger only ever sees one such name and needs no special case: whether |
| 985 | // it came from the synthetic mint or from a real route, it is the same record |
| 986 | // and the same SNI name. |
| 987 | // |
| 988 | // The prefix-only match here relies on steeredHosts' in-zone check |
| 989 | // (steering.go): every per-host record in want already lies under |
| 990 | // appsDomain, so an "edge."-prefixed Name reaching this function can only |
| 991 | // ever be the one true edge.<apps-domain> hostname, never some other |
| 992 | // zone's "edge.something-else.com" riding along by coincidence of prefix. |
| 993 | // That in-zone filter is what makes this match order-independent; it was |
| 994 | // added for a different reason (a steered host outside the zone wedges |
| 995 | // Route53's whole batch) but closes this hostname's own would-be |
| 996 | // ambiguity as a side effect. |
| 997 | func steeringHostname(want []desiredRecord) string { |
| 998 | for _, rec := range want { |
| 999 | if strings.HasPrefix(rec.Name, "edge.") { |
| 1000 | return strings.TrimSuffix(rec.Name, ".") |
| 1001 | } |
| 1002 | } |
| 1003 | return "" |
| 1004 | } |
| 1005 | |
| 1006 | // fallbackHostname picks an SNI name for eip when want carries no |
| 1007 | // edge.<apps-domain> record (no apps domain configured — steering.go logs |
| 1008 | // that case): the first wanted record's own name for that EIP. want's order |
| 1009 | // is deterministic (desiredState sorts and dedupes), so the choice is stable |
| 1010 | // across passes. |
| 1011 | func fallbackHostname(want []desiredRecord, eip string) string { |
| 1012 | for _, rec := range want { |
| 1013 | if rec.EIP == eip { |
| 1014 | return strings.TrimSuffix(rec.Name, ".") |
| 1015 | } |
| 1016 | } |
| 1017 | return "" |
| 1018 | } |
| 1019 | |
amadan