| index 417f82818..64d1cee5a 100644 |
| --- a/internal/registry/registry.go |
| +++ b/internal/registry/registry.go |
| @@ -75,6 +75,21 @@ type Route struct { |
| DBPath string |
| Version string |
| PrevVersion string |
| + // GoodVersion, GoodDigest and GoodDemand are the good build (zero-downtime |
| + // spec §8.5): the last build that served for this instance after passing |
| + // its health check, the sha256 of its staged executable, and the |
| + // release's own `swap` demand as verified then. Separate from |
| + // PrevVersion, which records install history whether or not the build |
| + // ever ran. Written only by SetGoodBuild, with an expected-current |
| + // condition; Upsert inserts them on a new row and never changes them on |
| + // an existing one. |
| + GoodVersion string |
| + GoodDigest string |
| + GoodDemand string |
| + // ServingVersion is the build of the generation that last took new |
| + // connections (spec §3), written by the agent for status and the CLI. |
| + // It is not cleared when the instance sleeps. |
| + ServingVersion string |
| |
| App string // which app's releases this route runs |
| Pool string // placement pool ("" = shared) |
| @@ -547,6 +562,10 @@ CREATE TABLE IF NOT EXISTS routes ( |
| `ALTER TABLE routes ADD COLUMN replicas TEXT NOT NULL DEFAULT ''`, |
| `ALTER TABLE routes ADD COLUMN replica INTEGER NOT NULL DEFAULT 0`, |
| `ALTER TABLE routes ADD COLUMN hold_op TEXT NOT NULL DEFAULT ''`, |
| + `ALTER TABLE routes ADD COLUMN good_version TEXT NOT NULL DEFAULT ''`, |
| + `ALTER TABLE routes ADD COLUMN good_digest TEXT NOT NULL DEFAULT ''`, |
| + `ALTER TABLE routes ADD COLUMN good_demand TEXT NOT NULL DEFAULT ''`, |
| + `ALTER TABLE routes ADD COLUMN serving_version TEXT NOT NULL DEFAULT ''`, |
| } { |
| if _, err := r.sql.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") { |
| return err |
| @@ -793,7 +812,7 @@ func (r *Registry) Accounts() ([]Account, error) { |
| } |
| |
| // routeCols is the column list every route read and the one route write |
| -// share — 47 names, in this exact order. scanRoute's destinations, Upsert's |
| +// share — 51 names, in this exact order. scanRoute's destinations, Upsert's |
| // placeholder row and Upsert's argument list all count from here: adding a |
| // column means adding it in FOUR places, and a miscount is a silent column |
| // shift rather than an error (TestUpsertRoundTripsAllColumns is what catches |
| @@ -803,7 +822,8 @@ version, prev_version, app, pool, region, channel, hibernate, canary, |
| replication, next_due, backing, status_named, platform, dns_status, |
| dns_checked_at, cert_expires_at, account, environment, caps, tunnel, |
| provisioned, seed, unit, fleet_box, addrs, health, steering, gate, created_at, |
| -args, idle_seconds, max_run_seconds, replicas, replica, claim_incarnation, claim_stamp, claim_owner, claim_row_id, hold_op` |
| +args, idle_seconds, max_run_seconds, replicas, replica, claim_incarnation, claim_stamp, claim_owner, claim_row_id, hold_op, |
| +good_version, good_digest, good_demand, serving_version` |
| |
| func scanRoute(scan func(...any) error) (Route, error) { |
| var t Route |
| @@ -814,7 +834,8 @@ func scanRoute(scan func(...any) error) (Route, error) { |
| &sn, &plat, &t.DNSStatus, &t.DNSCheckedAt, &t.CertExpiresAt, &t.Account, |
| &t.Environment, &t.Caps, &t.Tunnel, &prov, &t.Seed, &t.Unit, &t.FleetBox, |
| &t.Addrs, &t.Health, &t.Steering, &t.Gate, &t.CreatedAt, |
| - &t.Args, &t.IdleSeconds, &t.MaxRunSeconds, &t.Replicas, &rep, &t.ClaimIncarnation, &t.ClaimStamp, &t.ClaimOwner, &t.ClaimRowID, &t.HoldOp) |
| + &t.Args, &t.IdleSeconds, &t.MaxRunSeconds, &t.Replicas, &rep, &t.ClaimIncarnation, &t.ClaimStamp, &t.ClaimOwner, &t.ClaimRowID, &t.HoldOp, |
| + &t.GoodVersion, &t.GoodDigest, &t.GoodDemand, &t.ServingVersion) |
| t.Hibernate = hib != 0 |
| t.Canary = can != 0 |
| t.StatusNamed = sn != 0 |
| @@ -999,6 +1020,11 @@ func (r *Registry) Delete(host string) error { |
| // steering housekeeper duty (Task 6) is armed would silently clear a route's |
| // Route53 latency records — a real incident this repo caught in review before |
| // it shipped, not a hypothetical. |
| +// `good_version`, `good_digest`, `good_demand` and `serving_version` |
| +// (zero-downtime spec §8.5) are absent from the ON CONFLICT clause on |
| +// purpose: an existing row keeps them whatever the incoming Route carries. |
| +// They have one writer each (SetGoodBuild, SetServingVersion), and no |
| +// full-row caller knows what an instance last served. |
| func (r *Registry) Upsert(t Route) error { |
| if t.CreatedAt == 0 { |
| t.CreatedAt = time.Now().Unix() |
| @@ -1035,7 +1061,7 @@ func (r *Registry) Upsert(t Route) error { |
| } |
| res, err := r.sql.Exec(` |
| INSERT INTO routes (`+routeCols+`) |
| -VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) |
| +VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) |
| ON CONFLICT(host) DO UPDATE SET |
| slug=excluded.slug, kind=excluded.kind, socket=excluded.socket, |
| addr=excluded.addr, |
| @@ -1073,7 +1099,8 @@ AND routes.claim_owner=excluded.claim_owner AND routes.claim_row_id=excluded.cla |
| t.CertExpiresAt, t.Account, t.Environment, t.Caps, t.Tunnel, |
| b2i(t.Provisioned), t.Seed, t.Unit, t.FleetBox, t.Addrs, t.Health, |
| t.Steering, t.Gate, t.CreatedAt, t.Args, t.IdleSeconds, t.MaxRunSeconds, |
| - t.Replicas, b2i(t.Replica), t.ClaimIncarnation, t.ClaimStamp, t.ClaimOwner, t.ClaimRowID, t.HoldOp) |
| + t.Replicas, b2i(t.Replica), t.ClaimIncarnation, t.ClaimStamp, t.ClaimOwner, t.ClaimRowID, t.HoldOp, |
| + t.GoodVersion, t.GoodDigest, t.GoodDemand, t.ServingVersion) |
| if err != nil { |
| return err |
| } |
| @@ -1993,6 +2020,55 @@ func (r *Registry) SetVersions(host, ver, prev string) error { |
| return err |
| } |
| |
| +// GoodBuild is the good-build triple of zero-downtime spec §8.5. |
| +type GoodBuild struct{ Version, Digest, Demand string } |
| + |
| +// Good returns the route's good-build triple. |
| +func (t Route) Good() GoodBuild { |
| + return GoodBuild{Version: t.GoodVersion, Digest: t.GoodDigest, Demand: t.GoodDemand} |
| +} |
| + |
| +// ErrGoodBuildChanged is SetGoodBuild's answer when the row no longer holds |
| +// the triple the caller expected to replace. |
| +var ErrGoodBuildChanged = errors.New("registry: good build changed underneath this write") |
| + |
| +// SetGoodBuild replaces host's good build with next, but only if the row |
| +// still holds expect (spec §8.5: "written with an expected-current |
| +// condition"). Two writers that each saw a different generation become |
| +// current cannot both win; the loser learns what the row holds now. |
| +func (r *Registry) SetGoodBuild(host string, expect, next GoodBuild) error { |
| + res, err := r.sql.Exec(`UPDATE routes SET good_version = ?, good_digest = ?, good_demand = ? |
| +WHERE host = ? AND good_version = ? AND good_digest = ? AND good_demand = ?`, |
| + next.Version, next.Digest, next.Demand, host, expect.Version, expect.Digest, expect.Demand) |
| + if err != nil { |
| + return err |
| + } |
| + n, err := res.RowsAffected() |
| + if err != nil { |
| + return err |
| + } |
| + if n == 1 { |
| + return nil |
| + } |
| + var cur GoodBuild |
| + serr := r.sql.QueryRow(`SELECT good_version, good_digest, good_demand FROM routes WHERE host = ?`, host). |
| + Scan(&cur.Version, &cur.Digest, &cur.Demand) |
| + if serr == sql.ErrNoRows { |
| + return fmt.Errorf("no route for host %q", host) |
| + } |
| + if serr != nil { |
| + return serr |
| + } |
| + return fmt.Errorf("%w: %s now holds %+v", ErrGoodBuildChanged, host, cur) |
| +} |
| + |
| +// SetServingVersion records the build of the generation that just became |
| +// current (spec §3), as a narrow UPDATE. |
| +func (r *Registry) SetServingVersion(host, ver string) error { |
| + _, err := r.sql.Exec(`UPDATE routes SET serving_version = ? WHERE host = ?`, ver, host) |
| + return err |
| +} |
| + |
| // SetHibernate flips the per-host hibernation opt-in as a narrow UPDATE. |
| // Turning hibernation OFF also clears `replication`, and that is the point |
| // of doing it here rather than in each caller. The replication class is read |
| @@ -2665,9 +2741,9 @@ FROM route_paths ORDER BY host, prefix`) |
| // values are scanned generically and written in column order, which is why |
| // this needs no per-column typing the way Fingerprint does. |
| // |
| -// "Every column" has exactly two exceptions, snapshotChurnCols: the |
| -// scheduling stamps the box rewrites on its own clock with no operator |
| -// change behind them. Hashing those made the gate never hold (#207 — both |
| +// "Every column" has exactly three exceptions, snapshotChurnCols: the |
| +// scheduling stamps (and the serving_version rollout stamp) the box rewrites |
| +// on its own clock with no operator change behind them. Hashing those made the gate never hold (#207 — both |
| // production boxes wrote a generation on essentially every 5-minute tick, |
| // so the 8-generation history was ~40 minutes deep, not 8 changes deep). |
| // They are still IN the snapshot; only the decision to take one ignores |
| @@ -2691,8 +2767,9 @@ FROM route_paths ORDER BY host, prefix`) |
| // the box's own sweeps rewrite with nothing behind them an operator did. |
| // Keep this to columns that are re-derived within one sweep of a restore. |
| var snapshotChurnCols = map[string]bool{ |
| - "next_due": true, // activator scheduling (SetNextDue, every pass) |
| - "dns_checked_at": true, // DNS sweep stamp (SetDNSStatus, every sweep) |
| + "next_due": true, // activator scheduling (SetNextDue, every pass) |
| + "dns_checked_at": true, // DNS sweep stamp (SetDNSStatus, every sweep) |
| + "serving_version": true, // rollout stamp (SetServingVersion, every wake and swap) |
| } |
| |
| // snapshotClaimCols are the ownership columns, hashed only when non-default. |
| @@ -2703,6 +2780,17 @@ var snapshotClaimCols = map[string]bool{ |
| "claim_row_id": true, |
| } |
| |
| +// snapshotWhenSetCols are the good-build columns (zero-downtime spec §8.5). |
| +// A restore has to reproduce them, so they are hashed, but only when set and |
| +// tagged, on the hold_op rule: a registry that has never recorded a good |
| +// build digests exactly as it did before the columns existed, so upgrading |
| +// does not force a snapshot on every box. |
| +var snapshotWhenSetCols = map[string]bool{ |
| + "good_version": true, |
| + "good_digest": true, |
| + "good_demand": true, |
| +} |
| + |
| func (r *Registry) SnapshotFingerprint() (string, error) { |
| db, err := sql.Open("sqlite", r.path) |
| if err != nil { |
| @@ -2753,6 +2841,12 @@ func (r *Registry) SnapshotFingerprint() (string, error) { |
| } |
| continue |
| } |
| + if snapshotWhenSetCols[cols[i]] { |
| + if v.Valid && v.String != "" { |
| + fmt.Fprintf(h, "%s=%q\x1f", cols[i], v.String) |
| + } |
| + continue |
| + } |
| // NULL and "" must not hash alike: a column that has never been |
| // written is a different restore than one explicitly cleared. |
| if v.Valid { |
amadan