| 1 | package console |
| 2 | |
| 3 | // The relay's write side (relay spec §5). What is pinned here, in the order |
| 4 | // it would hurt if lost: an enable delivers a key AND publishes authority, or |
| 5 | // it does neither; the aggregate is rebuilt from records with the etag read |
| 6 | // FIRST, so a disable landing mid-listing fails the write instead of undoing |
| 7 | // itself; the reconcile converges an interrupted enable, a stale config write |
| 8 | // that outlived a disable, and an aggregate naming an app whose record is |
| 9 | // gone; and one console at a time holds the writer lease, with the loser |
| 10 | // stopping rather than writing. |
| 11 | |
| 12 | import ( |
| 13 | "context" |
| 14 | "encoding/base64" |
| 15 | "encoding/json" |
| 16 | "errors" |
| 17 | "net/http" |
| 18 | "net/url" |
| 19 | "strings" |
| 20 | "sync" |
| 21 | "testing" |
| 22 | "time" |
| 23 | |
| 24 | "github.com/carlosframework/platform/internal/appconfig" |
| 25 | "github.com/carlosframework/platform/internal/registry" |
| 26 | "github.com/carlosframework/platform/internal/relay" |
| 27 | "github.com/carlosframework/platform/internal/release" |
| 28 | ) |
| 29 | |
| 30 | // relayFixture is storeFixture (an account, the app titogo, a member's token) |
| 31 | // plus everything the relay needs: one armed region, a sealing key whose |
| 32 | // private half only the test holds, and a staged master in this console's own |
| 33 | // environment. It leaves the server HOLDING the writer lease, because every |
| 34 | // operation refuses without it and no test here is about that refusal. |
| 35 | // |
| 36 | // The private key comes back so a test can open the envelope the console |
| 37 | // delivered and prove it is the value the record implies — the console cannot |
| 38 | // do that, by design, so the assertion belongs on this side. |
| 39 | func relayFixture(t *testing.T, code string) (*consoleFixture, string) { |
| 40 | t.Helper() |
| 41 | fx, _ := storeFixture(t, code) |
| 42 | return fx, armRelay(t, fx) |
| 43 | } |
| 44 | |
| 45 | // armRelay gives any console fixture a relay: one armed region, a sealing key |
| 46 | // whose private half comes back to the caller, a staged master, and the |
| 47 | // writer lease held. |
| 48 | func armRelay(t *testing.T, fx *consoleFixture) string { |
| 49 | t.Helper() |
| 50 | pub, priv, _ := sealedFor(t, "irrelevant") |
| 51 | fx.srv.cfg.SecretsPub = pub |
| 52 | fx.srv.cfg.RelayRegions = []string{"eu-west-1"} |
| 53 | fx.srv.cfg.PlatformDomain = "oncarlos.com" |
| 54 | t.Setenv(relayMastersEnv("eu-west-1"), "k1:"+relayTestMaster(1)) |
| 55 | if err := fx.srv.relayAcquire(context.Background()); err != nil { |
| 56 | t.Fatalf("acquire the writer lease: %v", err) |
| 57 | } |
| 58 | return priv |
| 59 | } |
| 60 | |
| 61 | // relayTestMaster is a deterministic 32-byte master, base64 as the console |
| 62 | // variable carries it. |
| 63 | func relayTestMaster(seed byte) string { |
| 64 | b := make([]byte, relay.MasterKeyLen) |
| 65 | for i := range b { |
| 66 | b[i] = seed*31 + byte(i) |
| 67 | } |
| 68 | return base64.StdEncoding.EncodeToString(b) |
| 69 | } |
| 70 | |
| 71 | // account is the fixture's account as the operations take it. |
| 72 | func (fx *consoleFixture) account(t *testing.T) *Account { |
| 73 | t.Helper() |
| 74 | a, err := fx.st.AccountBySqid(fx.sqid) |
| 75 | if err != nil { |
| 76 | t.Fatalf("account %s: %v", fx.sqid, err) |
| 77 | } |
| 78 | return a |
| 79 | } |
| 80 | |
| 81 | // relayRecord reads the app's record straight out of the bucket. |
| 82 | func relayRecord(t *testing.T, fx *consoleFixture, app string) *relay.Record { |
| 83 | t.Helper() |
| 84 | rec, _, err := relay.GetRecord(context.Background(), fx.rel, fx.sqid, app) |
| 85 | if err != nil { |
| 86 | t.Fatalf("read relay record: %v", err) |
| 87 | } |
| 88 | return rec |
| 89 | } |
| 90 | |
| 91 | // relayAuthority reads the published authority record, an empty one when |
| 92 | // nothing has been published. |
| 93 | func relayAuthority(t *testing.T, fx *consoleFixture) *relay.Authority { |
| 94 | t.Helper() |
| 95 | a, _, err := relay.ReadAuthority(context.Background(), fx.rel) |
| 96 | if err != nil { |
| 97 | t.Fatalf("read authority: %v", err) |
| 98 | } |
| 99 | if a == nil { |
| 100 | return &relay.Authority{} |
| 101 | } |
| 102 | return a |
| 103 | } |
| 104 | |
| 105 | // TestRelayEnableWritesRecordConfigAndAggregate is the whole happy path: the |
| 106 | // record lands finished, the app's config carries a SEALED CARLOS_RELAY whose |
| 107 | // plaintext is the value the record implies, and the authority every relay |
| 108 | // box reads names the app at that incarnation. |
| 109 | func TestRelayEnableWritesRecordConfigAndAggregate(t *testing.T) { |
| 110 | fx, priv := relayFixture(t, "code-relay-enable") |
| 111 | ctx := context.Background() |
| 112 | |
| 113 | rec, err := fx.srv.relayEnable(ctx, fx.account(t), "titogo", nil, "alice@keymail.dev") |
| 114 | if err != nil { |
| 115 | t.Fatal(err) |
| 116 | } |
| 117 | if !rec.Enabled || rec.Phase != relayPhaseDone || rec.Op != 1 || len(rec.Incarnation) != 16 { |
| 118 | t.Fatalf("record = %+v", rec) |
| 119 | } |
| 120 | if got := relayRecord(t, fx, "titogo"); got == nil || got.Phase != relayPhaseDone || got.Incarnation != rec.Incarnation { |
| 121 | t.Fatalf("stored record = %+v, want the returned one finished", got) |
| 122 | } |
| 123 | |
| 124 | v, ok := appVars(t, fx, "titogo")[relayVar] |
| 125 | if !ok || v.Kind != "secret" || v.Envelope == "" { |
| 126 | t.Fatalf("%s = %+v, want a sealed secret", relayVar, v) |
| 127 | } |
| 128 | entries, err := relay.ParseConfig(openSecret(t, priv, v)) |
| 129 | if err != nil { |
| 130 | t.Fatalf("delivered value: %v", err) |
| 131 | } |
| 132 | if len(entries) != 1 || entries[0].Region != "eu-west-1" { |
| 133 | t.Fatalf("entries = %+v, want one for the armed region", entries) |
| 134 | } |
| 135 | wantURLs := []string{ |
| 136 | "stun:relay.eu-west-1.oncarlos.com:3478", |
| 137 | "turn:relay.eu-west-1.oncarlos.com:3478?transport=udp", |
| 138 | "turn:relay.eu-west-1.oncarlos.com:3478?transport=tcp", |
| 139 | } |
| 140 | if strings.Join(entries[0].URLs, ",") != strings.Join(wantURLs, ",") { |
| 141 | t.Fatalf("urls = %v, want %v", entries[0].URLs, wantURLs) |
| 142 | } |
| 143 | wantSuffix := relay.Suffix(fx.sqid, "titogo", rec.Incarnation, "k1") |
| 144 | if entries[0].UsernameSuffix != wantSuffix || rec.IssuedSuffixes["eu-west-1"] != wantSuffix { |
| 145 | t.Fatalf("suffix = %q / recorded %q, want %q", entries[0].UsernameSuffix, rec.IssuedSuffixes["eu-west-1"], wantSuffix) |
| 146 | } |
| 147 | |
| 148 | a := relayAuthority(t, fx) |
| 149 | x, ok := a.Lookup(fx.sqid, "titogo") |
| 150 | if !ok || x.Incarnation != rec.Incarnation || strings.Join(x.Regions, ",") != "eu-west-1" { |
| 151 | t.Fatalf("aggregate = %+v", a) |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | // TestRelayRepeatEnableKeepsIncarnationAndWritesNoConfig: enabling an app |
| 156 | // that is already enabled with the same regions and nothing owed writes |
| 157 | // NOTHING — no op bump, no new envelope, no generation touch. A member who |
| 158 | // re-runs the command must not re-key their own app. |
| 159 | func TestRelayRepeatEnableKeepsIncarnationAndWritesNoConfig(t *testing.T) { |
| 160 | fx, _ := relayFixture(t, "code-relay-repeat") |
| 161 | ctx := context.Background() |
| 162 | |
| 163 | first, err := fx.srv.relayEnable(ctx, fx.account(t), "titogo", nil, "alice@keymail.dev") |
| 164 | if err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | before := appVars(t, fx, "titogo")[relayVar] |
| 168 | _, genBefore, err := fx.rel.Get(ctx, appconfig.GenKey(fx.sqid, "titogo")) |
| 169 | if err != nil { |
| 170 | t.Fatal(err) |
| 171 | } |
| 172 | |
| 173 | second, err := fx.srv.relayEnable(ctx, fx.account(t), "titogo", []string{"eu-west-1"}, "alice@keymail.dev") |
| 174 | if err != nil { |
| 175 | t.Fatal(err) |
| 176 | } |
| 177 | if second.Incarnation != first.Incarnation || second.Op != first.Op { |
| 178 | t.Fatalf("second enable = %+v, want the same incarnation and op as %+v", second, first) |
| 179 | } |
| 180 | if after := appVars(t, fx, "titogo")[relayVar]; after.Envelope != before.Envelope { |
| 181 | t.Fatal("a repeat enable re-sealed the value — the app would re-key for nothing") |
| 182 | } |
| 183 | _, genAfter, err := fx.rel.Get(ctx, appconfig.GenKey(fx.sqid, "titogo")) |
| 184 | if err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | if genAfter != genBefore { |
| 188 | t.Fatalf("config generation moved (%q -> %q) with no config write", genBefore, genAfter) |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | // TestRelayDisableThenEnableIsANewIncarnation: a disable takes the app out of |
| 193 | // authority and takes the variable out of its config; the enable after it |
| 194 | // draws a NEW incarnation, so a credential minted before the disable can |
| 195 | // never be mistaken for a live one. |
| 196 | func TestRelayDisableThenEnableIsANewIncarnation(t *testing.T) { |
| 197 | fx, _ := relayFixture(t, "code-relay-cycle") |
| 198 | ctx := context.Background() |
| 199 | acct := fx.account(t) |
| 200 | |
| 201 | first, err := fx.srv.relayEnable(ctx, acct, "titogo", nil, "alice@keymail.dev") |
| 202 | if err != nil { |
| 203 | t.Fatal(err) |
| 204 | } |
| 205 | off, err := fx.srv.relayDisable(ctx, acct, "titogo", "alice@keymail.dev") |
| 206 | if err != nil { |
| 207 | t.Fatal(err) |
| 208 | } |
| 209 | if off.Enabled || off.Phase != relayPhaseDone || off.Op != first.Op+1 { |
| 210 | t.Fatalf("disabled record = %+v", off) |
| 211 | } |
| 212 | if _, ok := appVars(t, fx, "titogo")[relayVar]; ok { |
| 213 | t.Fatalf("%s survived the disable", relayVar) |
| 214 | } |
| 215 | if _, ok := relayAuthority(t, fx).Lookup(fx.sqid, "titogo"); ok { |
| 216 | t.Fatal("the aggregate still names a disabled app") |
| 217 | } |
| 218 | |
| 219 | again, err := fx.srv.relayEnable(ctx, acct, "titogo", nil, "alice@keymail.dev") |
| 220 | if err != nil { |
| 221 | t.Fatal(err) |
| 222 | } |
| 223 | if again.Incarnation == first.Incarnation { |
| 224 | t.Fatalf("re-enable kept incarnation %s — a pre-disable key would still verify", again.Incarnation) |
| 225 | } |
| 226 | if again.Op != off.Op+1 { |
| 227 | t.Fatalf("op = %d after a re-enable, want %d (the counter never rewinds)", again.Op, off.Op+1) |
| 228 | } |
| 229 | if x, ok := relayAuthority(t, fx).Lookup(fx.sqid, "titogo"); !ok || x.Incarnation != again.Incarnation { |
| 230 | t.Fatalf("aggregate = %+v, want the new incarnation", x) |
| 231 | } |
| 232 | } |
| 233 | |
| 234 | // relayOpStore records the ORDER of the store calls one publish makes — the |
| 235 | // only way to prove the etag is read before the listing rather than after, |
| 236 | // which is the whole safety of spec §5's publish sequence. |
| 237 | type relayOpStore struct { |
| 238 | release.Store |
| 239 | mu sync.Mutex |
| 240 | ops []string |
| 241 | } |
| 242 | |
| 243 | func (r *relayOpStore) note(op string) { |
| 244 | r.mu.Lock() |
| 245 | defer r.mu.Unlock() |
| 246 | r.ops = append(r.ops, op) |
| 247 | } |
| 248 | |
| 249 | func (r *relayOpStore) seq() []string { |
| 250 | r.mu.Lock() |
| 251 | defer r.mu.Unlock() |
| 252 | return append([]string(nil), r.ops...) |
| 253 | } |
| 254 | |
| 255 | func (r *relayOpStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 256 | r.note("get " + key) |
| 257 | return r.Store.Get(ctx, key) |
| 258 | } |
| 259 | |
| 260 | func (r *relayOpStore) ListDirs(ctx context.Context, prefix string) ([]string, error) { |
| 261 | r.note("listdirs " + prefix) |
| 262 | return r.Store.ListDirs(ctx, prefix) |
| 263 | } |
| 264 | |
| 265 | func (r *relayOpStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| 266 | r.note("put " + key) |
| 267 | return r.Store.Put(ctx, key, body, ifMatch) |
| 268 | } |
| 269 | |
| 270 | // TestRelayPublishAggregateIsEtagFirst pins the sequence itself: the |
| 271 | // aggregate's etag is read BEFORE any record is listed, and a publish carrying |
| 272 | // an etag that has since moved fails with ErrCASConflict rather than |
| 273 | // overwriting whoever moved it. The retry then publishes the current truth. |
| 274 | func TestRelayPublishAggregateIsEtagFirst(t *testing.T) { |
| 275 | fx, _ := relayFixture(t, "code-relay-etag") |
| 276 | ctx := context.Background() |
| 277 | acct := fx.account(t) |
| 278 | |
| 279 | if _, err := fx.srv.relayEnable(ctx, acct, "titogo", nil, "alice@keymail.dev"); err != nil { |
| 280 | t.Fatal(err) |
| 281 | } |
| 282 | if _, err := fx.srv.relayEnable(ctx, acct, "meet", nil, "alice@keymail.dev"); err != nil { |
| 283 | t.Fatal(err) |
| 284 | } |
| 285 | |
| 286 | // The order, on a publish with nothing else going on. |
| 287 | rec := &relayOpStore{Store: fx.rel} |
| 288 | fx.srv.cfg.Releases = rec |
| 289 | if err := fx.srv.relayPublishAggregate(ctx); err != nil { |
| 290 | t.Fatal(err) |
| 291 | } |
| 292 | fx.srv.cfg.Releases = rec.Store |
| 293 | seq := rec.seq() |
| 294 | getAt, listAt := -1, -1 |
| 295 | for i, op := range seq { |
| 296 | if getAt < 0 && op == "get "+relay.AuthorityKey { |
| 297 | getAt = i |
| 298 | } |
| 299 | if listAt < 0 && strings.HasPrefix(op, "listdirs apps/") { |
| 300 | listAt = i |
| 301 | } |
| 302 | } |
| 303 | if getAt < 0 || listAt < 0 || getAt > listAt { |
| 304 | t.Fatalf("publish sequence = %v, want the aggregate's etag read before the listing", seq) |
| 305 | } |
| 306 | |
| 307 | // A stale etag must lose. Take one, then move the aggregate underneath it |
| 308 | // by disabling titogo's record directly and republishing. |
| 309 | _, stale, err := fx.rel.Get(ctx, relay.AuthorityKey) |
| 310 | if err != nil { |
| 311 | t.Fatal(err) |
| 312 | } |
| 313 | off, offEtag, err := relay.GetRecord(ctx, fx.rel, fx.sqid, "titogo") |
| 314 | if err != nil { |
| 315 | t.Fatal(err) |
| 316 | } |
| 317 | off.Enabled = false |
| 318 | if _, err := relay.PutRecord(ctx, fx.rel, fx.sqid, "titogo", off, offEtag); err != nil { |
| 319 | t.Fatal(err) |
| 320 | } |
| 321 | if err := fx.srv.relayPublishAggregate(ctx); err != nil { |
| 322 | t.Fatal(err) |
| 323 | } |
| 324 | if err := fx.srv.relayPublishOnce(ctx, stale); !errors.Is(err, release.ErrCASConflict) { |
| 325 | t.Fatalf("publish with a stale etag = %v, want ErrCASConflict", err) |
| 326 | } |
| 327 | if err := fx.srv.relayPublishAggregate(ctx); err != nil { |
| 328 | t.Fatal(err) |
| 329 | } |
| 330 | a := relayAuthority(t, fx) |
| 331 | if _, ok := a.Lookup(fx.sqid, "titogo"); ok { |
| 332 | t.Fatalf("aggregate = %+v, want the disabled app gone", a) |
| 333 | } |
| 334 | if _, ok := a.Lookup(fx.sqid, "meet"); !ok { |
| 335 | t.Fatalf("aggregate = %+v, want the still-enabled app kept", a) |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | // TestRelayReconcileFinishesInterruptedEnable: a console that wrote the |
| 340 | // record and died before the config write leaves an enabled record at phase |
| 341 | // `config`, no variable and no authority. The next holder's reconcile |
| 342 | // delivers both and marks it done — without the member running anything. |
| 343 | func TestRelayReconcileFinishesInterruptedEnable(t *testing.T) { |
| 344 | fx, _ := relayFixture(t, "code-relay-resume") |
| 345 | ctx := context.Background() |
| 346 | inc := relay.NewIncarnation() |
| 347 | if _, err := relay.PutRecord(ctx, fx.rel, fx.sqid, "titogo", &relay.Record{ |
| 348 | Enabled: true, Incarnation: inc, Regions: []string{"eu-west-1"}, |
| 349 | Op: 1, Phase: relayPhaseConfig, EnabledAt: time.Now().UTC(), |
| 350 | }, ""); err != nil { |
| 351 | t.Fatal(err) |
| 352 | } |
| 353 | |
| 354 | if err := fx.srv.relayReconcile(ctx); err != nil { |
| 355 | t.Fatal(err) |
| 356 | } |
| 357 | |
| 358 | if v, ok := appVars(t, fx, "titogo")[relayVar]; !ok || v.Kind != "secret" { |
| 359 | t.Fatalf("%s = %+v after the reconcile, want a delivered secret", relayVar, v) |
| 360 | } |
| 361 | if x, ok := relayAuthority(t, fx).Lookup(fx.sqid, "titogo"); !ok || x.Incarnation != inc { |
| 362 | t.Fatal("the reconcile did not publish the interrupted enable") |
| 363 | } |
| 364 | if got := relayRecord(t, fx, "titogo"); got.Phase != relayPhaseDone { |
| 365 | t.Fatalf("phase = %q after the reconcile, want done", got.Phase) |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | // TestRelayReconcileUndoesStaleConfigAfterDisable: a config write that lands |
| 370 | // AFTER a disable — a losing writer's last act — is undone within one |
| 371 | // reconcile. This is the convergence spec §5 promises for the case the phase |
| 372 | // walk alone cannot cover, because the record is already finished. |
| 373 | func TestRelayReconcileUndoesStaleConfigAfterDisable(t *testing.T) { |
| 374 | fx, _ := relayFixture(t, "code-relay-stale") |
| 375 | ctx := context.Background() |
| 376 | acct := fx.account(t) |
| 377 | |
| 378 | if _, err := fx.srv.relayEnable(ctx, acct, "titogo", nil, "alice@keymail.dev"); err != nil { |
| 379 | t.Fatal(err) |
| 380 | } |
| 381 | if _, err := fx.srv.relayDisable(ctx, acct, "titogo", "alice@keymail.dev"); err != nil { |
| 382 | t.Fatal(err) |
| 383 | } |
| 384 | // The straggler write. |
| 385 | if _, err := appconfig.Apply(ctx, fx.rel, fx.sqid, "titogo", |
| 386 | map[string]appconfig.Var{relayVar: {Kind: "secret", Envelope: "stale-envelope"}}, nil); err != nil { |
| 387 | t.Fatal(err) |
| 388 | } |
| 389 | |
| 390 | if err := fx.srv.relayReconcile(ctx); err != nil { |
| 391 | t.Fatal(err) |
| 392 | } |
| 393 | if v, ok := appVars(t, fx, "titogo")[relayVar]; ok { |
| 394 | t.Fatalf("%s = %+v, want it removed again", relayVar, v) |
| 395 | } |
| 396 | } |
| 397 | |
| 398 | // TestRelayReconcileRepairsDeletionWithoutRecord: an app whose record is gone |
| 399 | // — deleted with the rest of its prefix by a walk that died before publishing |
| 400 | // — disappears from authority on the next unconditional rebuild. That is why |
| 401 | // the reconcile lists records rather than hunting for unfinished ones. |
| 402 | func TestRelayReconcileRepairsDeletionWithoutRecord(t *testing.T) { |
| 403 | fx, _ := relayFixture(t, "code-relay-deleted") |
| 404 | ctx := context.Background() |
| 405 | body, err := json.Marshal(relay.Authority{ |
| 406 | Apps: []relay.AuthorityApp{{Account: fx.sqid, App: "gone", |
| 407 | Incarnation: relay.NewIncarnation(), Regions: []string{"eu-west-1"}}}, |
| 408 | WrittenAt: time.Now().UTC(), |
| 409 | }) |
| 410 | if err != nil { |
| 411 | t.Fatal(err) |
| 412 | } |
| 413 | if _, err := fx.rel.Put(ctx, relay.AuthorityKey, body, ""); err != nil { |
| 414 | t.Fatal(err) |
| 415 | } |
| 416 | |
| 417 | if err := fx.srv.relayReconcile(ctx); err != nil { |
| 418 | t.Fatal(err) |
| 419 | } |
| 420 | if _, ok := relayAuthority(t, fx).Lookup(fx.sqid, "gone"); ok { |
| 421 | t.Fatal("the aggregate still names an app with no record") |
| 422 | } |
| 423 | } |
| 424 | |
| 425 | // TestRelayPurgeRepublishesTheAggregate: permanently deleting an app takes it |
| 426 | // out of every relay's authority AT ONCE, not within five minutes. The |
| 427 | // deletion walk sweeps the record; publishing right after is what makes the |
| 428 | // app's credentials stop verifying while the operator is still watching. |
| 429 | func TestRelayPurgeRepublishesTheAggregate(t *testing.T) { |
| 430 | fx, c, csrf := trashPageFixture(t) |
| 431 | armRelay(t, fx) |
| 432 | ctx := context.Background() |
| 433 | |
| 434 | if _, err := fx.srv.relayEnable(ctx, fx.account(t), "kass", nil, "paul@keymail.dev"); err != nil { |
| 435 | t.Fatal(err) |
| 436 | } |
| 437 | if _, ok := relayAuthority(t, fx).Lookup(fx.sqid, "kass"); !ok { |
| 438 | t.Fatal("the enable did not publish") |
| 439 | } |
| 440 | |
| 441 | status, _, body := postGated(t, fx, c, fx.u("/trash/delete"), |
| 442 | url.Values{"csrf": {csrf}, "app": {"kass"}, "confirm": {"kass"}}) |
| 443 | if status != http.StatusSeeOther { |
| 444 | t.Fatalf("purge = %d:\n%s", status, body) |
| 445 | } |
| 446 | if keys := appKeys(t, fx, "kass"); len(keys) != 0 { |
| 447 | t.Fatalf("the bucket still holds %v", keys) |
| 448 | } |
| 449 | // No reconcile anywhere in this test: the purge itself republished. |
| 450 | if _, ok := relayAuthority(t, fx).Lookup(fx.sqid, "kass"); ok { |
| 451 | t.Fatal("a purged app still holds relay authority") |
| 452 | } |
| 453 | } |
| 454 | |
| 455 | // TestRelayEnableRefusesRouteWithoutDelivery: a TCP-address route this |
| 456 | // platform does not own, whose bundle has not opted into file delivery, could |
| 457 | // never pick up a rotated key — so it would block every future retirement |
| 458 | // forever. Enable refuses it BY NAME, and writes nothing on the way out. |
| 459 | func TestRelayEnableRefusesRouteWithoutDelivery(t *testing.T) { |
| 460 | fx, _ := relayFixture(t, "code-relay-delivery") |
| 461 | ctx := context.Background() |
| 462 | fx.addRoute(t, registry.Route{Host: "x.titogo.oncarlos.com", App: "titogo", |
| 463 | Kind: "service", Addr: "10.0.0.5:8080", Status: "ready"}) |
| 464 | |
| 465 | _, err := fx.srv.relayEnable(ctx, fx.account(t), "titogo", nil, "alice@keymail.dev") |
| 466 | if !errors.Is(err, errRelayUndelivered) { |
| 467 | t.Fatalf("enable over an undeliverable route = %v, want a refusal", err) |
| 468 | } |
| 469 | if !strings.Contains(err.Error(), "x.titogo.oncarlos.com") { |
| 470 | t.Fatalf("error = %q, want the route named", err) |
| 471 | } |
| 472 | if rec := relayRecord(t, fx, "titogo"); rec != nil { |
| 473 | t.Fatalf("a refused enable left a record: %+v", rec) |
| 474 | } |
| 475 | if _, ok := appVars(t, fx, "titogo")[relayVar]; ok { |
| 476 | t.Fatal("a refused enable delivered a key") |
| 477 | } |
| 478 | |
| 479 | // Opting the app in makes the same route deliverable. |
| 480 | if err := appconfig.SetExecDelivery(ctx, fx.rel, fx.sqid, "titogo", appconfig.ExecDeliveryFile); err != nil { |
| 481 | t.Fatal(err) |
| 482 | } |
| 483 | if _, err := fx.srv.relayEnable(ctx, fx.account(t), "titogo", nil, "alice@keymail.dev"); err != nil { |
| 484 | t.Fatalf("enable after opting into file delivery: %v", err) |
| 485 | } |
| 486 | } |
| 487 | |
| 488 | // TestRelayAdoptedRecordsTheIssuedIdentitiesAndRefusesAStranger: a member's |
| 489 | // ack stamps the identities issued AT THAT MOMENT against their own route, so |
| 490 | // an ack given before a later rotation cannot count for it — and a host that |
| 491 | // is not a route of this app is refused rather than appended, which is what |
| 492 | // keeps the record bounded. |
| 493 | // |
| 494 | // The route is a tcp-exec-file one because that is the only shape a member |
| 495 | // may acknowledge at all: anything the config sync restarts itself has its |
| 496 | // cutover observed (TestAPIRelayAdoptedRefusesARouteThePlatformRestarts). |
| 497 | func TestRelayAdoptedRecordsTheIssuedIdentitiesAndRefusesAStranger(t *testing.T) { |
| 498 | fx, _ := relayFixture(t, "code-relay-adopted") |
| 499 | ctx := context.Background() |
| 500 | acct := fx.account(t) |
| 501 | fx.addRoute(t, registry.Route{Host: "titogo.oncarlos.com", App: "titogo", |
| 502 | Kind: "service", Status: "ready", Addr: "10.0.0.9:8080"}) |
| 503 | if err := appconfig.SetExecDelivery(ctx, fx.rel, fx.sqid, "titogo", appconfig.ExecDeliveryFile); err != nil { |
| 504 | t.Fatal(err) |
| 505 | } |
| 506 | |
| 507 | rec, err := fx.srv.relayEnable(ctx, acct, "titogo", nil, "alice@keymail.dev") |
| 508 | if err != nil { |
| 509 | t.Fatal(err) |
| 510 | } |
| 511 | if _, err := fx.srv.relayAdopted(ctx, acct, "titogo", "somewhere.else.example", "alice@keymail.dev"); !errors.Is(err, errRelayUnknownHost) { |
| 512 | t.Fatalf("adopted for a host that is not a route = %v, want a refusal", err) |
| 513 | } |
| 514 | after, err := fx.srv.relayAdopted(ctx, acct, "titogo", "titogo.oncarlos.com", "alice@keymail.dev") |
| 515 | if err != nil { |
| 516 | t.Fatal(err) |
| 517 | } |
| 518 | if len(after.Cutovers) != 1 || after.Cutovers[0].Host != "titogo.oncarlos.com" || |
| 519 | after.Cutovers[0].Suffixes["eu-west-1"] != rec.IssuedSuffixes["eu-west-1"] { |
| 520 | t.Fatalf("cutovers = %+v, want one naming the issued identity", after.Cutovers) |
| 521 | } |
| 522 | // A second ack for the same route replaces the first rather than growing |
| 523 | // the record. |
| 524 | again, err := fx.srv.relayAdopted(ctx, acct, "titogo", "titogo.oncarlos.com", "alice@keymail.dev") |
| 525 | if err != nil || len(again.Cutovers) != 1 { |
| 526 | t.Fatalf("second ack = %+v (%v), want one cutover still", again.Cutovers, err) |
| 527 | } |
| 528 | } |
| 529 | |
| 530 | // TestRelayRotateRefusesUntilHeartbeatsCarryKid: rotation's whole safety is |
| 531 | // that the console will not issue from a master a live relay has not loaded. |
| 532 | // Until the region's heartbeat lists the new kid, rotate refuses and the |
| 533 | // issuer pointer does not move; once it does, every enabled app is re-issued |
| 534 | // under the new kid. |
| 535 | func TestRelayRotateRefusesUntilHeartbeatsCarryKid(t *testing.T) { |
| 536 | fx, _ := relayFixture(t, "code-relay-rotate") |
| 537 | ctx := context.Background() |
| 538 | t.Setenv(relayMastersEnv("eu-west-1"), "k1:"+relayTestMaster(1)+",k2:"+relayTestMaster(2)) |
| 539 | |
| 540 | before, err := fx.srv.relayEnable(ctx, fx.account(t), "titogo", nil, "alice@keymail.dev") |
| 541 | if err != nil { |
| 542 | t.Fatal(err) |
| 543 | } |
| 544 | if !strings.HasSuffix(before.IssuedSuffixes["eu-west-1"], ":k1") { |
| 545 | t.Fatalf("issued %q, want the first staged master", before.IssuedSuffixes["eu-west-1"]) |
| 546 | } |
| 547 | hb := relay.Heartbeat{Box: "box1", Region: "eu-west-1", ReadAt: time.Now(), |
| 548 | Listeners: []string{"udp:3478"}, Kids: []string{"k1"}} |
| 549 | if err := relay.WriteHeartbeat(ctx, fx.rel, hb); err != nil { |
| 550 | t.Fatal(err) |
| 551 | } |
| 552 | |
| 553 | if _, err := fx.srv.relayRotate(ctx, "eu-west-1"); err == nil || |
| 554 | !strings.Contains(err.Error(), "k2") || !strings.Contains(err.Error(), "box1") { |
| 555 | t.Fatalf("rotate before the relay loaded the kid = %v, want a refusal naming the box and kid", err) |
| 556 | } |
| 557 | if issuers, _, err := fx.srv.relayIssuers(ctx); err != nil || issuers["eu-west-1"] != "" { |
| 558 | t.Fatalf("issuer pointer = %v (%v) after a refused rotation, want it untouched", issuers, err) |
| 559 | } |
| 560 | |
| 561 | hb.Kids = []string{"k1", "k2"} |
| 562 | if err := relay.WriteHeartbeat(ctx, fx.rel, hb); err != nil { |
| 563 | t.Fatal(err) |
| 564 | } |
| 565 | res, err := fx.srv.relayRotate(ctx, "eu-west-1") |
| 566 | if err != nil { |
| 567 | t.Fatal(err) |
| 568 | } |
| 569 | if res.Kid != "k2" || res.Apps != 1 { |
| 570 | t.Fatalf("rotate result = %+v, want kid k2 and one app re-issued", res) |
| 571 | } |
| 572 | // A SECOND rotate in the same window must refuse. The staged variable |
| 573 | // still holds "k1,k2" — the old master stays there for the 28 hours before |
| 574 | // it may be retired — and rotating "to the entry that is not the issuer" |
| 575 | // would walk the pointer back onto the master about to be retired. |
| 576 | if _, err := fx.srv.relayRotate(ctx, "eu-west-1"); err == nil || |
| 577 | !strings.Contains(err.Error(), "already the issuer") { |
| 578 | t.Fatalf("re-rotate with nothing new staged = %v, want a refusal naming the last entry", err) |
| 579 | } |
| 580 | if issuers, _, err := fx.srv.relayIssuers(ctx); err != nil || issuers["eu-west-1"] != "k2" { |
| 581 | t.Fatalf("issuer pointer = %v (%v) after a refused re-rotate, want it still on k2", issuers, err) |
| 582 | } |
| 583 | if d := time.Until(res.RetireNoEarlierThan); d < 27*time.Hour || d > 29*time.Hour { |
| 584 | t.Fatalf("retire-no-earlier-than is %s away, want about 28h", d) |
| 585 | } |
| 586 | issuers, _, err := fx.srv.relayIssuers(ctx) |
| 587 | if err != nil || issuers["eu-west-1"] != "k2" { |
| 588 | t.Fatalf("issuer pointer = %v (%v), want k2", issuers, err) |
| 589 | } |
| 590 | after := relayRecord(t, fx, "titogo") |
| 591 | if !strings.HasSuffix(after.IssuedSuffixes["eu-west-1"], ":k2") { |
| 592 | t.Fatalf("issued %q after the rotation, want the new kid", after.IssuedSuffixes["eu-west-1"]) |
| 593 | } |
| 594 | if after.Incarnation != before.Incarnation { |
| 595 | t.Fatal("a rotation changed the incarnation — every live allocation would end") |
| 596 | } |
| 597 | if after.Op != before.Op+1 || after.Phase != relayPhaseDone { |
| 598 | t.Fatalf("record = %+v, want one finished operation past %d", after, before.Op) |
| 599 | } |
| 600 | |
| 601 | // Staging a THIRD master, with the pointer on the middle one, rotates |
| 602 | // forward to it — the last entry, not merely a different one. |
| 603 | t.Setenv(relayMastersEnv("eu-west-1"), |
| 604 | "k1:"+relayTestMaster(1)+",k2:"+relayTestMaster(2)+",k3:"+relayTestMaster(3)) |
| 605 | hb.Kids = []string{"k1", "k2", "k3"} |
| 606 | if err := relay.WriteHeartbeat(ctx, fx.rel, hb); err != nil { |
| 607 | t.Fatal(err) |
| 608 | } |
| 609 | third, err := fx.srv.relayRotate(ctx, "eu-west-1") |
| 610 | if err != nil { |
| 611 | t.Fatal(err) |
| 612 | } |
| 613 | if third.Kid != "k3" { |
| 614 | t.Fatalf("rotate with k1,k2,k3 staged and the pointer on k2 = %q, want k3", third.Kid) |
| 615 | } |
| 616 | if !strings.HasSuffix(relayRecord(t, fx, "titogo").IssuedSuffixes["eu-west-1"], ":k3") { |
| 617 | t.Fatal("the app was not re-issued under the newest kid") |
| 618 | } |
| 619 | } |
| 620 | |
| 621 | // TestRelayReconcileRepairsOverwrittenEnvelope: the bundle holding SOMETHING |
| 622 | // under CARLOS_RELAY is not the same fact as it holding what this console |
| 623 | // wrote. A straggler write from a superseded console, or a member setting the |
| 624 | // variable themselves, leaves suffixes that still agree and a value that does |
| 625 | // not — which is exactly what the record's issued digest catches. |
| 626 | func TestRelayReconcileRepairsOverwrittenEnvelope(t *testing.T) { |
| 627 | fx, priv := relayFixture(t, "code-relay-envelope") |
| 628 | ctx := context.Background() |
| 629 | |
| 630 | rec, err := fx.srv.relayEnable(ctx, fx.account(t), "titogo", nil, "alice@keymail.dev") |
| 631 | if err != nil { |
| 632 | t.Fatal(err) |
| 633 | } |
| 634 | if rec.IssuedDigest == "" { |
| 635 | t.Fatal("enable recorded no issued digest") |
| 636 | } |
| 637 | // Someone else's envelope, under the same name, with the record untouched. |
| 638 | _, _, foreign := sealedFor(t, "not what the console issued") |
| 639 | if _, err := appconfig.Apply(ctx, fx.rel, fx.sqid, "titogo", |
| 640 | map[string]appconfig.Var{relayVar: {Kind: "secret", Envelope: foreign}}, nil); err != nil { |
| 641 | t.Fatal(err) |
| 642 | } |
| 643 | |
| 644 | if err := fx.srv.relayReconcile(ctx); err != nil { |
| 645 | t.Fatal(err) |
| 646 | } |
| 647 | |
| 648 | v := appVars(t, fx, "titogo")[relayVar] |
| 649 | if v.Envelope == foreign { |
| 650 | t.Fatal("the reconcile left an envelope this console did not write") |
| 651 | } |
| 652 | after := relayRecord(t, fx, "titogo") |
| 653 | if after.IssuedDigest != relayEnvelopeDigest(v.Envelope) { |
| 654 | t.Fatalf("record digest %q does not describe the bundle's envelope", after.IssuedDigest) |
| 655 | } |
| 656 | entries, err := relay.ParseConfig(openSecret(t, priv, v)) |
| 657 | if err != nil || len(entries) != 1 || entries[0].UsernameSuffix != after.IssuedSuffixes["eu-west-1"] { |
| 658 | t.Fatalf("re-delivered value = %+v (%v), want the record's own identity", entries, err) |
| 659 | } |
| 660 | } |
| 661 | |
| 662 | // TestRelayWriterLeaseExcludesSecondHolder: two consoles over one bucket, one |
| 663 | // lease. The second is excluded until the first's minute lapses — and the |
| 664 | // first, once it has been taken over, fails its next renewal and drops the |
| 665 | // flag every relay write checks, which is what stops a superseded writer |
| 666 | // finishing a walk the new holder has already started. |
| 667 | func TestRelayWriterLeaseExcludesSecondHolder(t *testing.T) { |
| 668 | fx, _ := relayFixture(t, "code-relay-lease") |
| 669 | ctx := context.Background() |
| 670 | |
| 671 | other := newConsoleFixture(t) |
| 672 | other.srv.cfg.Releases = fx.rel |
| 673 | other.srv.cfg.RelayRegions = []string{"eu-west-1"} |
| 674 | |
| 675 | base := time.Now() |
| 676 | first, second := base, base.Add(time.Second) |
| 677 | fx.srv.relayNow = func() time.Time { return first } |
| 678 | other.srv.relayNow = func() time.Time { return second } |
| 679 | |
| 680 | if err := fx.srv.relayAcquire(ctx); err != nil || !fx.srv.relayHold() { |
| 681 | t.Fatalf("first acquire = %v, held = %v", err, fx.srv.relayHold()) |
| 682 | } |
| 683 | err := other.srv.relayAcquire(ctx) |
| 684 | if !errors.Is(err, errRelayNotHeld) { |
| 685 | t.Fatalf("second acquire inside the TTL = %v, want a refusal", err) |
| 686 | } |
| 687 | if other.srv.relayHold() { |
| 688 | t.Fatal("the second console thinks it holds the lease") |
| 689 | } |
| 690 | if err := other.srv.relayReconcile(ctx); !errors.Is(err, errRelayNotHeld) { |
| 691 | t.Fatalf("reconcile without the lease = %v, want a refusal", err) |
| 692 | } |
| 693 | |
| 694 | // Past the expiry, the lease is takeable. |
| 695 | second = base.Add(relayLeaseTTL + time.Second) |
| 696 | if err := other.srv.relayAcquire(ctx); err != nil || !other.srv.relayHold() { |
| 697 | t.Fatalf("acquire after expiry = %v, held = %v", err, other.srv.relayHold()) |
| 698 | } |
| 699 | |
| 700 | // And the first console's next renewal fails, dropping its own flag. |
| 701 | first = base.Add(relayLeaseTTL + 2*time.Second) |
| 702 | if err := fx.srv.relayAcquire(ctx); err == nil { |
| 703 | t.Fatal("the superseded console renewed a lease it does not hold") |
| 704 | } |
| 705 | if fx.srv.relayHold() { |
| 706 | t.Fatal("the superseded console still believes it holds the lease") |
| 707 | } |
| 708 | if _, err := fx.srv.relayEnable(ctx, fx.account(t), "titogo", nil, "alice@keymail.dev"); !errors.Is(err, errRelayNotHeld) { |
| 709 | t.Fatalf("enable without the lease = %v, want a refusal", err) |
| 710 | } |
| 711 | } |
| 712 | |
amadan