| 1 | // Config adoption, observed per route (relay spec §5). Config sync writes |
| 2 | // an additive acknowledgement after it converges a route: the shape it |
| 3 | // delivered into, whether a restart happened and succeeded, and — for a |
| 4 | // route whose rendered CARLOS_RELAY carries relay config — the identities |
| 5 | // it just wrote. Nothing reads the ack today but `carlos relay status` |
| 6 | // (internal/console/apirelay.go); it is the first per-route statement of |
| 7 | // "this is what the process has" the platform keeps. |
| 8 | // |
| 9 | // The ack names IDENTITIES, not kids: RelaySuffixes carries each region's |
| 10 | // full username_suffix (account:app:incarnation:kid), parsed from the |
| 11 | // rendered CARLOS_RELAY value with relay.ParseConfig. A kid alone would not |
| 12 | // do — a disable and re-enable between two host syncs changes the |
| 13 | // incarnation and the key while keeping the kid, and an ack naming only |
| 14 | // kids would report the old credentials as adopted. |
| 15 | // |
| 16 | // internal/appconfig importing internal/relay is one-directional and safe: |
| 17 | // internal/relay does not import internal/appconfig. |
| 18 | package appconfig |
| 19 | |
| 20 | import ( |
| 21 | "context" |
| 22 | "crypto/ecdh" |
| 23 | "encoding/json" |
| 24 | "errors" |
| 25 | "fmt" |
| 26 | "log" |
| 27 | "os" |
| 28 | "strings" |
| 29 | "time" |
| 30 | |
| 31 | "github.com/carlosframework/platform/internal/registry" |
| 32 | "github.com/carlosframework/platform/internal/relay" |
| 33 | "github.com/carlosframework/platform/internal/release" |
| 34 | ) |
| 35 | |
| 36 | // ConfigAckSchema is the ack schema THIS binary writes. It exists so a |
| 37 | // reader can tell an agent that renders the instance config layer (senders |
| 38 | // design §6.2) from one that does not, and the only honest way to do that |
| 39 | // is a positive statement: an ack written by an older agent decodes with |
| 40 | // Schema 0, because the field was not there to write. |
| 41 | // |
| 42 | // It is not the manifest schema and is not tied to it. Bump it when an |
| 43 | // agent starts stating something a reader may need to depend on. |
| 44 | const ConfigAckSchema = 1 |
| 45 | |
| 46 | // instanceConfigETagSchema is the ack schema at which InstanceConfigETag |
| 47 | // first became a field an agent could write — 1, the schema this plan |
| 48 | // introduced it at. InstanceConfigDelivery compares against THIS constant, |
| 49 | // not against ConfigAckSchema: the two happen to be equal today, but they |
| 50 | // answer different questions, and conflating them is a trap the next schema |
| 51 | // bump would fall into silently. ConfigAckSchema is "what schema does this |
| 52 | // BINARY write" and is meant to climb every time the ack states something |
| 53 | // new; instanceConfigETagSchema is "at what schema did THIS field become |
| 54 | // trustworthy" and has no reason to climb again unless the field itself is |
| 55 | // ever redefined. A bump to ConfigAckSchema for some unrelated addition |
| 56 | // (say, schema 2 adds a field about something else entirely) must not |
| 57 | // demote every schema-1 ack — which carries a perfectly good |
| 58 | // InstanceConfigETag — to InstanceConfigUnknownAgent for a question it can |
| 59 | // already answer. |
| 60 | const instanceConfigETagSchema = 1 |
| 61 | |
| 62 | // ConfigAck is one route's config-adoption acknowledgement, written after |
| 63 | // config sync converges its env/config file — see ConfigAckKey for where. |
| 64 | type ConfigAck struct { |
| 65 | Box string `json:"box"` |
| 66 | Host string `json:"host"` |
| 67 | // Schema is ConfigAckSchema as written by the agent that produced this |
| 68 | // ack, and 0 for every ack written before this field existed. A reader |
| 69 | // asking "does this box render the instance config layer" reads this and |
| 70 | // nothing else: absence of an ack, or an ack at schema 0, both mean the |
| 71 | // question cannot be answered from the box's own statements yet. |
| 72 | Schema int `json:"schema"` |
| 73 | // Shape is one of "unit", "socket-exec" or "tcp-exec-file" — spec §5's |
| 74 | // three delivery shapes. |
| 75 | Shape string `json:"shape"` |
| 76 | At time.Time `json:"at"` |
| 77 | // Restarted and RestartError describe the restart/bounce this sync pass |
| 78 | // attempted for the route, if it could attempt one at all: a unit route |
| 79 | // that try-restarted cleanly sets Restarted true; one whose try-restart |
| 80 | // failed sets RestartError instead. syncExecHost (the exec-file and |
| 81 | // socket-exec shapes) always leaves both zero — its bounce is |
| 82 | // best-effort and fire-and-forget (bounceLogged swallows the error), |
| 83 | // and a TCP-exec-file route's process belongs to an external supervisor |
| 84 | // this platform cannot restart at all; its cutover is a member |
| 85 | // acknowledgement instead (carlos relay adopted). |
| 86 | Restarted bool `json:"restarted"` |
| 87 | RestartError string `json:"restart_error,omitempty"` |
| 88 | // Hibernating is set when the route was asleep at sync time, so its |
| 89 | // restart (a systemd try-restart on an inactive unit) was a no-op by |
| 90 | // construction — the config landed but nothing woke to run it. `carlos |
| 91 | // relay status` reports such a route as "hibernating, delivered on |
| 92 | // wake" rather than claiming an adoption that has not happened yet. |
| 93 | Hibernating bool `json:"hibernating,omitempty"` |
| 94 | // RelaySuffixes is the username_suffix (account:app:incarnation:kid) |
| 95 | // this pass rendered into CARLOS_RELAY, per region — nil if the route's |
| 96 | // manifest carries no CARLOS_RELAY entry at all. |
| 97 | RelaySuffixes map[string]string `json:"relay_suffixes,omitempty"` |
| 98 | // InstanceConfigETag is the etag of the instance overlay |
| 99 | // (apps/<acct>/<app>/instances-config/<host>.json) this pass rendered |
| 100 | // into the file, empty when the route has no overlay at all. The writer |
| 101 | // keeps the etag its own Put returned and compares the two |
| 102 | // (InstanceConfigDelivery): equal is delivered, different is stale. A |
| 103 | // map of delivered var names could not make that distinction — an agent |
| 104 | // too old to read the layer and a new agent rendering an overlay whose |
| 105 | // vars the reader does not recognise produce the same map. |
| 106 | InstanceConfigETag string `json:"instance_config_etag,omitempty"` |
| 107 | // MoveOp, FileSHA256 and BundleETags are written only by a move's held |
| 108 | // render (RenderHeld, moves design §9.1), never by a sync pass. MoveOp is |
| 109 | // the operation the render was made for, FileSHA256 the hex sha256 of the |
| 110 | // file as it stands after the render, and BundleETags the etags of the |
| 111 | // "default" and "environment" objects merged into it ("" for one that was |
| 112 | // absent or not bound). T3 compares them with what it read itself. |
| 113 | MoveOp string `json:"move_op,omitempty"` |
| 114 | FileSHA256 string `json:"file_sha256,omitempty"` |
| 115 | BundleETags map[string]string `json:"bundle_etags,omitempty"` |
| 116 | } |
| 117 | |
| 118 | // ConfigAckPrefix is where every route's config-adoption ack lives, one |
| 119 | // object per (box, route host). |
| 120 | const ConfigAckPrefix = "control/config-ack/" |
| 121 | |
| 122 | // ConfigAckKey is where a route's config-adoption ack lives: additive, one |
| 123 | // object per (box, route host), rewritten (never appended to) on every sync |
| 124 | // that changes what was delivered. |
| 125 | func ConfigAckKey(box, host string) string { |
| 126 | return ConfigAckPrefix + box + "/" + host + ".json" |
| 127 | } |
| 128 | |
| 129 | // The four answers to "is this instance running the overlay I wrote", named |
| 130 | // because three of them are not "no" and a caller that flattened them would |
| 131 | // tell an operator to roll a box that is merely one pass behind. |
| 132 | const ( |
| 133 | // InstanceConfigDelivered: the agent rendered exactly this version of |
| 134 | // the overlay. a route with no overlay, whose caller expects none, is |
| 135 | // also delivered. there is nothing outstanding. |
| 136 | InstanceConfigDelivered = "delivered" |
| 137 | // InstanceConfigStale: the agent rendered a different version. one more |
| 138 | // pass, or a generation that never got read, is the usual cause. |
| 139 | InstanceConfigStale = "stale" |
| 140 | // InstanceConfigUnknownAgent: this box's agent predates the instance |
| 141 | // config layer, so its ack says nothing about it. not "no". the answer |
| 142 | // cannot be had until the box is rolled. |
| 143 | InstanceConfigUnknownAgent = "unknown-agent" |
| 144 | // InstanceConfigNotYetDelivered: no ack for this host at all. acks are |
| 145 | // change-only, so this is the honest answer for a route whose file has |
| 146 | // not moved since its agent rolled. it never means "old agent": that |
| 147 | // question is the box record's agent version, which step 5 reads |
| 148 | // (design §6.5). |
| 149 | InstanceConfigNotYetDelivered = "not-yet-delivered" |
| 150 | ) |
| 151 | |
| 152 | // InstanceConfigDelivery compares an ack with the etag a write returned and |
| 153 | // says which of the four states the route is in. want is the etag of the |
| 154 | // overlay the caller wrote, or "" when the caller wrote none. |
| 155 | // |
| 156 | // the comparison is on etags rather than on delivered var names because only |
| 157 | // etags are decidable: an agent too old to read the layer and a new agent |
| 158 | // rendering an overlay whose vars the reader does not recognise deliver the |
| 159 | // same set of names (design §6.5). |
| 160 | // |
| 161 | // The schema test below is against instanceConfigETagSchema, deliberately |
| 162 | // NOT ConfigAckSchema — see that constant's own doc comment for the hazard |
| 163 | // a future schema bump would otherwise walk into. |
| 164 | func InstanceConfigDelivery(a *ConfigAck, want string) string { |
| 165 | switch { |
| 166 | case a == nil: |
| 167 | return InstanceConfigNotYetDelivered |
| 168 | case a.Schema < instanceConfigETagSchema: |
| 169 | return InstanceConfigUnknownAgent |
| 170 | case a.InstanceConfigETag == want: |
| 171 | return InstanceConfigDelivered |
| 172 | default: |
| 173 | return InstanceConfigStale |
| 174 | } |
| 175 | } |
| 176 | |
| 177 | // LatestConfigAck reads the freshest ack naming host, across every box that |
| 178 | // has written one, and nil (with no error) when nothing has. a route can |
| 179 | // legitimately have acks from two boxes at once across a migration, and the |
| 180 | // freshest is the one describing what is running now. |
| 181 | // |
| 182 | // it lists the whole ack prefix, which is the same limit relayConfigAcks |
| 183 | // accepts (internal/console/relayops.go): there is no per-host box list to |
| 184 | // narrow a listing by. the Gets are narrowed, though. only keys whose |
| 185 | // basename is this host are read. |
| 186 | func LatestConfigAck(ctx context.Context, s release.Store, host string) (*ConfigAck, error) { |
| 187 | keys, err := s.List(ctx, ConfigAckPrefix) |
| 188 | if err != nil { |
| 189 | return nil, err |
| 190 | } |
| 191 | var best *ConfigAck |
| 192 | for _, key := range keys { |
| 193 | if !strings.HasSuffix(key, "/"+host+".json") { |
| 194 | continue |
| 195 | } |
| 196 | body, _, err := s.Get(ctx, key) |
| 197 | if err != nil { |
| 198 | if errors.Is(err, os.ErrNotExist) { |
| 199 | continue |
| 200 | } |
| 201 | return nil, err |
| 202 | } |
| 203 | var a ConfigAck |
| 204 | if err := json.Unmarshal(body, &a); err != nil { |
| 205 | // one unreadable object must not hide the box that answered |
| 206 | // properly, and there is nothing in it to report either way. |
| 207 | log.Printf("config: config ack %s: %v", key, err) |
| 208 | continue |
| 209 | } |
| 210 | if best == nil || a.At.After(best.At) { |
| 211 | latest := a |
| 212 | best = &latest |
| 213 | } |
| 214 | } |
| 215 | return best, nil |
| 216 | } |
| 217 | |
| 218 | // WriteConfigAck writes ack, get-then-put with the CAS etag of whatever is |
| 219 | // there now — "" (create) only when nothing is. An ack is rewritten on |
| 220 | // every sync that changes delivery, unlike release.Store.Put's create-only |
| 221 | // contract, so this can't just Put with "": the object almost always |
| 222 | // already exists. Retries once on release.ErrCASConflict, the same shape as |
| 223 | // relay.WriteHeartbeat (internal/relay/authority.go) — a conflict here can |
| 224 | // only be this route's own previous ack, since ConfigAckKey is unique per |
| 225 | // (box, host) and nothing else writes it. |
| 226 | func WriteConfigAck(ctx context.Context, s release.Store, ack ConfigAck) error { |
| 227 | body, err := json.Marshal(ack) |
| 228 | if err != nil { |
| 229 | return err |
| 230 | } |
| 231 | key := ConfigAckKey(ack.Box, ack.Host) |
| 232 | var lastErr error |
| 233 | for attempt := 0; attempt < 2; attempt++ { |
| 234 | etag := "" |
| 235 | _, curEtag, err := s.Get(ctx, key) |
| 236 | switch { |
| 237 | case errors.Is(err, os.ErrNotExist): |
| 238 | // no existing record: create-only Put below. |
| 239 | case err != nil: |
| 240 | return err |
| 241 | default: |
| 242 | etag = curEtag |
| 243 | } |
| 244 | if _, err := s.Put(ctx, key, body, etag); err != nil { |
| 245 | lastErr = err |
| 246 | if errors.Is(err, release.ErrCASConflict) { |
| 247 | continue |
| 248 | } |
| 249 | return err |
| 250 | } |
| 251 | return nil |
| 252 | } |
| 253 | return lastErr |
| 254 | } |
| 255 | |
| 256 | // carlosRelayVar is the well-known var name relay enable's config write |
| 257 | // (spec §5 step 5) uses — appconfig.Apply, same as any other app var. |
| 258 | const carlosRelayVar = "CARLOS_RELAY" |
| 259 | |
| 260 | // relaySuffixesFromRelayVar decrypts m's CARLOS_RELAY var, if it has one, |
| 261 | // and parses it with relay.ParseConfig into a region -> username_suffix |
| 262 | // map. Reads the decrypted manifest var directly — the same "env" vs |
| 263 | // "secret" decoding RenderEnv applies to every var — never the rendered |
| 264 | // EnvironmentFile bytes: CARLOS_RELAY is exactly as likely to be a sealed |
| 265 | // secret entry as any other var, and re-parsing rendered KEY=value lines |
| 266 | // would just repeat, worse, the decryption RenderEnv already performs. |
| 267 | // |
| 268 | // Returns nil, nil when the manifest carries no CARLOS_RELAY entry at all |
| 269 | // — an ordinary route the relay was never enabled for, not an error. |
| 270 | func relaySuffixesFromRelayVar(m *Manifest, priv *ecdh.PrivateKey, b Binding) (map[string]string, error) { |
| 271 | v, ok := m.Vars[carlosRelayVar] |
| 272 | if !ok { |
| 273 | return nil, nil |
| 274 | } |
| 275 | var val string |
| 276 | switch v.Kind { |
| 277 | case "env": |
| 278 | val = v.Value |
| 279 | case "secret": |
| 280 | if priv == nil { |
| 281 | return nil, fmt.Errorf("var %s: secret entry but no decryption key was supplied", carlosRelayVar) |
| 282 | } |
| 283 | plain, err := Open(priv, b, v.Envelope) |
| 284 | if err != nil { |
| 285 | return nil, fmt.Errorf("decrypt %s: %w", carlosRelayVar, err) |
| 286 | } |
| 287 | val = string(plain) |
| 288 | default: |
| 289 | return nil, fmt.Errorf("var %s: unknown kind %q", carlosRelayVar, v.Kind) |
| 290 | } |
| 291 | entries, err := relay.ParseConfig(val) |
| 292 | if err != nil { |
| 293 | return nil, fmt.Errorf("%s: %w", carlosRelayVar, err) |
| 294 | } |
| 295 | suffixes := make(map[string]string, len(entries)) |
| 296 | for _, e := range entries { |
| 297 | suffixes[e.Region] = e.UsernameSuffix |
| 298 | } |
| 299 | return suffixes, nil |
| 300 | } |
| 301 | |
| 302 | // shapeOf names which of spec §5's three delivery shapes rt is: "unit" for |
| 303 | // a carlos-app@<host> unit tenant, "socket-exec" for an activator-spawned |
| 304 | // route the platform bounces over a unix socket, "tcp-exec-file" for a |
| 305 | // TCP-address route belonging to a process this platform does not own (the |
| 306 | // config lands as a file; adoption is a member acknowledgement instead of |
| 307 | // a restart). Shared by syncHost and syncExecHost so the two never name a |
| 308 | // shape differently for the same route. |
| 309 | func shapeOf(rt registry.Route) string { |
| 310 | switch { |
| 311 | case rt.Backing == registry.BackingUnit: |
| 312 | return "unit" |
| 313 | case rt.Socket != "": |
| 314 | return "socket-exec" |
| 315 | default: |
| 316 | return "tcp-exec-file" |
| 317 | } |
| 318 | } |
| 319 | |
amadan