| 1 | // Package-file doc: control/changes.json is the change index (S3 change-index |
| 2 | // design §3): a map from change scope to a generation number that every |
| 3 | // successful bucket mutation inside that scope bumps. A box that has read the |
| 4 | // index can compare one small object's etag, and the generations behind it, |
| 5 | // instead of re-walking the deployment bucket. Like control/serial it is |
| 6 | // UNSIGNED and carries no authority: it can only say "something may have |
| 7 | // changed", so a reader that cannot trust it walks as before. |
| 8 | package release |
| 9 | |
| 10 | import ( |
| 11 | "context" |
| 12 | "encoding/json" |
| 13 | "errors" |
| 14 | "fmt" |
| 15 | "log" |
| 16 | "math/rand/v2" |
| 17 | "os" |
| 18 | "strings" |
| 19 | "time" |
| 20 | ) |
| 21 | |
| 22 | // ChangeIndexKey sits under control/ next to SerialKey, outside every prefix |
| 23 | // that app or account listings walk. |
| 24 | const ChangeIndexKey = "control/changes.json" |
| 25 | |
| 26 | // ErrChangeIndexSchema is an index written by a newer schema than this build |
| 27 | // understands. Readers treat it as unknown (walk as today); writers refuse to |
| 28 | // overwrite it. |
| 29 | var ErrChangeIndexSchema = errors.New("release: change index schema unsupported") |
| 30 | |
| 31 | const changeIndexAttempts = 4 |
| 32 | |
| 33 | // ChangeIndex is the body of ChangeIndexKey. Generations only increase and |
| 34 | // entries are never removed in v1. |
| 35 | type ChangeIndex struct { |
| 36 | Schema int `json:"schema"` |
| 37 | Entries map[string]uint64 `json:"entries"` |
| 38 | } |
| 39 | |
| 40 | // dataPrefixes are the per-app prefixes that hold high-volume data rather |
| 41 | // than state a box loop consumes; writes there never bump a scope. |
| 42 | // schedule-status is each box's report of its timers, written on every |
| 43 | // scheduled fire and read only by the console's schedule door, never by a |
| 44 | // box loop: bumping on it would run a scoped pass on every armed box per fire. |
| 45 | var dataPrefixes = map[string]bool{ |
| 46 | "logs": true, "errors": true, "analytics": true, "runs": true, "metrics": true, |
| 47 | "schedule-status": true, |
| 48 | } |
| 49 | |
| 50 | // ChangeScope maps a bucket key to the scope whose generation a write to it |
| 51 | // bumps. The scopes are apps/<a>/<b> (except the data prefixes), accounts/<a>, |
| 52 | // control/certs and control/edges; every other key reports ok=false. |
| 53 | func ChangeScope(key string) (scope string, ok bool) { |
| 54 | parts := strings.Split(key, "/") |
| 55 | switch parts[0] { |
| 56 | case "apps": |
| 57 | // apps/<a>/<b>/<rest...>: all three segments non-empty. |
| 58 | if len(parts) < 4 || parts[1] == "" || parts[2] == "" || parts[3] == "" || dataPrefixes[parts[3]] { |
| 59 | return "", false |
| 60 | } |
| 61 | return "apps/" + parts[1] + "/" + parts[2], true |
| 62 | case "accounts": |
| 63 | if len(parts) < 3 || parts[1] == "" || parts[2] == "" { |
| 64 | return "", false |
| 65 | } |
| 66 | return "accounts/" + parts[1], true |
| 67 | case "control": |
| 68 | if len(parts) < 3 || parts[2] == "" { |
| 69 | return "", false |
| 70 | } |
| 71 | if parts[1] == "certs" || parts[1] == "edges" { |
| 72 | return "control/" + parts[1], true |
| 73 | } |
| 74 | } |
| 75 | return "", false |
| 76 | } |
| 77 | |
| 78 | // ReadChangeIndex returns the index and its etag for CAS. An absent object is |
| 79 | // the ordinary pre-first-bump state: an empty schema-1 index and etag "". A |
| 80 | // schema other than 1 is ErrChangeIndexSchema and an unparseable body is an |
| 81 | // error too; neither is ever reported as empty, because "empty" would read as |
| 82 | // "nothing changed". |
| 83 | func ReadChangeIndex(ctx context.Context, s Store) (ChangeIndex, string, error) { |
| 84 | return readChangeIndex(ctx, s.Get) |
| 85 | } |
| 86 | |
| 87 | func readChangeIndex(ctx context.Context, get func(context.Context, string) ([]byte, string, error)) (ChangeIndex, string, error) { |
| 88 | body, etag, err := get(ctx, ChangeIndexKey) |
| 89 | if errors.Is(err, os.ErrNotExist) { |
| 90 | return ChangeIndex{Schema: 1, Entries: map[string]uint64{}}, "", nil |
| 91 | } |
| 92 | if err != nil { |
| 93 | return ChangeIndex{}, "", err |
| 94 | } |
| 95 | var idx ChangeIndex |
| 96 | if err := json.Unmarshal(body, &idx); err != nil { |
| 97 | return ChangeIndex{}, "", fmt.Errorf("release: change index malformed: %w", err) |
| 98 | } |
| 99 | if idx.Schema != 1 { |
| 100 | return ChangeIndex{}, "", fmt.Errorf("%w: schema %d", ErrChangeIndexSchema, idx.Schema) |
| 101 | } |
| 102 | if idx.Entries == nil { |
| 103 | idx.Entries = map[string]uint64{} |
| 104 | } |
| 105 | return idx, etag, nil |
| 106 | } |
| 107 | |
| 108 | // bumpChangeIndex increments every distinct scope's generation in one read-modify-CAS, |
| 109 | // retrying a lost race up to four times with 20-120ms of jitter between |
| 110 | // attempts. It takes raw get/put funcs rather than a Store so the stores' |
| 111 | // own mutation methods can call it with their unhooked internals. It makes no |
| 112 | // context of its own: the caller supplies the (detached, time-boxed) one. |
| 113 | // Callers treat failure as log-only; a lost bump costs latency, never |
| 114 | // correctness, and must never fail the write that triggered it. |
| 115 | func bumpChangeIndex(ctx context.Context, |
| 116 | get func(context.Context, string) ([]byte, string, error), |
| 117 | put func(context.Context, string, []byte, string) (string, error), |
| 118 | scopes ...string) error { |
| 119 | var lastErr error |
| 120 | for attempt := 0; attempt < changeIndexAttempts; attempt++ { |
| 121 | if attempt > 0 { |
| 122 | jitter := 20*time.Millisecond + rand.N(100*time.Millisecond) |
| 123 | t := time.NewTimer(jitter) |
| 124 | select { |
| 125 | case <-ctx.Done(): |
| 126 | t.Stop() |
| 127 | return ctx.Err() |
| 128 | case <-t.C: |
| 129 | } |
| 130 | } |
| 131 | idx, etag, err := readChangeIndex(ctx, get) |
| 132 | if err != nil { |
| 133 | return err |
| 134 | } |
| 135 | seen := make(map[string]bool, len(scopes)) |
| 136 | for _, sc := range scopes { |
| 137 | if !seen[sc] { // one write is one generation per scope, however many keys it touched |
| 138 | seen[sc] = true |
| 139 | idx.Entries[sc]++ |
| 140 | } |
| 141 | } |
| 142 | body, err := json.Marshal(idx) |
| 143 | if err != nil { |
| 144 | return err |
| 145 | } |
| 146 | _, err = put(ctx, ChangeIndexKey, body, etag) |
| 147 | if err == nil { |
| 148 | return nil |
| 149 | } |
| 150 | if !errors.Is(err, ErrCASConflict) { |
| 151 | return err |
| 152 | } |
| 153 | lastErr = err |
| 154 | } |
| 155 | return fmt.Errorf("bump %s: lost CAS race %d times: %w", ChangeIndexKey, changeIndexAttempts, lastErr) |
| 156 | } |
| 157 | |
| 158 | // changeIndexBumpBudget bounds one post-mutation bump, retries included. A |
| 159 | // var so tests can shrink it. |
| 160 | var changeIndexBumpBudget = 2 * time.Second |
| 161 | |
| 162 | // bumpAfterMutation is the hook both concrete stores call after a successful |
| 163 | // mutation of key. It does nothing unless the store enabled the index and key |
| 164 | // falls in a change scope. It runs under a context detached from the caller's |
| 165 | // cancellation (a request that returns right after its write must not drop the |
| 166 | // bump) but bounded by changeIndexBumpBudget, and it takes the stores' raw, |
| 167 | // unhooked get/put so writing the index never bumps again. A failure is logged |
| 168 | // and swallowed: the write already happened. |
| 169 | func bumpAfterMutation(ctx context.Context, enabled bool, key string, |
| 170 | get func(context.Context, string) ([]byte, string, error), |
| 171 | put func(context.Context, string, []byte, string) (string, error)) { |
| 172 | if !enabled { |
| 173 | return |
| 174 | } |
| 175 | scope, ok := ChangeScope(key) |
| 176 | if !ok { |
| 177 | return |
| 178 | } |
| 179 | bctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), changeIndexBumpBudget) |
| 180 | defer cancel() |
| 181 | if err := bumpChangeIndex(bctx, get, put, scope); err != nil { |
| 182 | log.Printf("release: change index bump %s: %v", scope, err) |
| 183 | } |
| 184 | } |
| 185 | |
amadan