| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os" |
| 7 | "path/filepath" |
| 8 | "strings" |
| 9 | "sync" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/hostops" |
| 14 | "github.com/carlosframework/platform/internal/release" |
| 15 | ) |
| 16 | |
| 17 | // I2-2: a console database restored onto a replacement instance opens under |
| 18 | // a new database ID. Effects bound to the original ID can never settle here. |
| 19 | func TestPlan9Fix2RestoreOntoReplacementInstanceAdoptsNewIdentity(t *testing.T) { |
| 20 | path := filepath.Join(t.TempDir(), "console.db") |
| 21 | openAs := func(instance string) (*Store, string) { |
| 22 | t.Helper() |
| 23 | restore := UseSQLInstanceIdentityForTest(instance) |
| 24 | defer restore() |
| 25 | s, err := OpenGated(path) |
| 26 | if err != nil { |
| 27 | t.Fatalf("open on %s: %v", instance, err) |
| 28 | } |
| 29 | id, err := s.sqlDatabaseID() |
| 30 | if err != nil { |
| 31 | s.Close() |
| 32 | t.Fatal(err) |
| 33 | } |
| 34 | return s, id |
| 35 | } |
| 36 | s, original := openAs("i-original") |
| 37 | s.Close() |
| 38 | |
| 39 | replacement, adopted := openAs("i-replacement") |
| 40 | if adopted == original || !strings.Contains(adopted, ":i-replacement:") { |
| 41 | replacement.Close() |
| 42 | t.Fatalf("replacement did not adopt a new identity: original %q, adopted %q", original, adopted) |
| 43 | } |
| 44 | nonce := func(id string) string { return strings.SplitN(id, ":", 2)[0] } |
| 45 | if nonce(adopted) == nonce(original) { |
| 46 | replacement.Close() |
| 47 | t.Fatalf("adoption kept the original nonce %q", nonce(original)) |
| 48 | } |
| 49 | // The original's effects stay unsettled: a fence for them is refused. |
| 50 | sc := hostops.SQLReceiptScope{Effect: "e1", Operation: "op1", AppID: "app", Epoch: 1, Database: original} |
| 51 | if _, err := replacement.FenceSQLOutcome(t.Context(), sc); !errors.Is(err, hostops.ErrIdentity) { |
| 52 | replacement.Close() |
| 53 | t.Fatalf("replacement fenced an effect bound to the original database: %v", err) |
| 54 | } |
| 55 | replacement.Close() |
| 56 | |
| 57 | // The adopted identity is stable across restarts on the replacement. |
| 58 | again, id := openAs("i-replacement") |
| 59 | again.Close() |
| 60 | if id != adopted { |
| 61 | t.Fatalf("adopted identity changed on reopen: %q then %q", adopted, id) |
| 62 | } |
| 63 | // Moving the copy back to the original instance is another adoption, not |
| 64 | // a return to the original identity. |
| 65 | back, id := openAs("i-original") |
| 66 | back.Close() |
| 67 | if id == original || id == adopted { |
| 68 | t.Fatalf("copy moved back reused an earlier identity: %q", id) |
| 69 | } |
| 70 | // m-3: the same file on the same instance keeps its ID (a block-level |
| 71 | // restore looks like this). The runbook's break-glass step copies it to |
| 72 | // a new file and moves that into place, which gives it a new ID. |
| 73 | same, sameID := openAs("i-original") |
| 74 | same.Close() |
| 75 | if sameID != id { |
| 76 | t.Fatalf("same file on the same instance changed identity: %q then %q", id, sameID) |
| 77 | } |
| 78 | b, err := os.ReadFile(path) |
| 79 | if err != nil { |
| 80 | t.Fatal(err) |
| 81 | } |
| 82 | if err = os.WriteFile(path+".copy", b, 0o600); err != nil { |
| 83 | t.Fatal(err) |
| 84 | } |
| 85 | if err = os.Rename(path+".copy", path); err != nil { |
| 86 | t.Fatal(err) |
| 87 | } |
| 88 | copied, copiedID := openAs("i-original") |
| 89 | copied.Close() |
| 90 | if copiedID == id { |
| 91 | t.Fatalf("a file copy moved into place kept the identity %q", id) |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | // replacedFixture is a gated console whose sweep sees one SQL effect past |
| 96 | // sql_recover_after bound to database, with two operators of whom only |
| 97 | // ops-a has a keymail grant. |
| 98 | func replacedFixture(t *testing.T, st *Store, database string) (*Server, *[]string) { |
| 99 | t.Helper() |
| 100 | raw := release.NewDirStore(t.TempDir()) |
| 101 | w := claimTestWriter(t, raw) |
| 102 | reg := hostops.Registration{Kind: hostops.RegistrationIngest, Owner: claimTestOwner, AppID: "app", Epoch: 1, Expires: time.Now().Add(time.Hour)} |
| 103 | past := time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano) |
| 104 | seedClaimAdmission(t, raw, "bab", "demo", hostops.AppAdmission{V: 1, AppID: "app", Revision: 1, Epoch: 1, State: "open", |
| 105 | Registrations: map[string]hostops.Registration{"old": reg}, |
| 106 | Effects: map[string]hostops.Effect{"old": {ID: "old", OperationID: "old", AppID: "app", Resource: "app-sql:old", Action: "sql-ingest", Owner: claimTestOwner, State: "uncertain", |
| 107 | Inputs: map[string]string{"sql_database": database, "sql_recover_after": past}}}}) |
| 108 | for _, addr := range []string{"ops-a@keymail.dev", "ops-b@keymail.dev"} { |
| 109 | if err := st.SetOperator(addr, true); err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | } |
| 113 | if err := st.SetGrantToken("ops-a@keymail.dev", "grant-a", time.Now()); err != nil { |
| 114 | t.Fatal(err) |
| 115 | } |
| 116 | var mu sync.Mutex |
| 117 | sent := &[]string{} |
| 118 | s := &Server{store: st, cfg: Config{HostClaimOps: &hostops.Controller{Store: raw, Control: w, Now: time.Now}}} |
| 119 | s.sendApp = func(_ context.Context, _, to, _, _ string) (string, error) { |
| 120 | mu.Lock() |
| 121 | defer mu.Unlock() |
| 122 | *sent = append(*sent, to) |
| 123 | return "msg", nil |
| 124 | } |
| 125 | return s, sent |
| 126 | } |
| 127 | |
| 128 | // m-4: the alert send is bounded by the sweep's own context, not a fresh |
| 129 | // background deadline per recipient. |
| 130 | func TestPlan9Fix2ReplacedAlertSendHonoursSweepContext(t *testing.T) { |
| 131 | st := openGatedStore(t) |
| 132 | s, _ := replacedFixture(t, st, "restored-away") |
| 133 | for _, addr := range []string{"ops-b@keymail.dev", "ops-c@keymail.dev"} { |
| 134 | if err := st.SetOperator(addr, true); err != nil { |
| 135 | t.Fatal(err) |
| 136 | } |
| 137 | if err := st.SetGrantToken(addr, "grant-"+addr, time.Now()); err != nil { |
| 138 | t.Fatal(err) |
| 139 | } |
| 140 | } |
| 141 | s.sendApp = func(ctx context.Context, _, _, _, _ string) (string, error) { |
| 142 | <-ctx.Done() |
| 143 | return "", ctx.Err() |
| 144 | } |
| 145 | ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) |
| 146 | defer cancel() |
| 147 | start := time.Now() |
| 148 | err := s.sweepOneClaimAdmission(ctx, hostops.App{Account: "bab", Name: "demo"}) |
| 149 | if !errors.Is(err, ErrSQLDatabaseReplaced) { |
| 150 | t.Fatalf("sweep: %v", err) |
| 151 | } |
| 152 | if took := time.Since(start); took > 5*time.Second { |
| 153 | t.Fatalf("a stalled keymail send held the serial sweep for %v", took) |
| 154 | } |
| 155 | } |
| 156 | |
| 157 | // m-4: a transient failure to read the recipients is retried on the next |
| 158 | // sweep instead of being recorded as "nobody to tell". |
| 159 | func TestPlan9Fix2ReplacedAlertRetriesAfterRecipientFailure(t *testing.T) { |
| 160 | st := openGatedStore(t) |
| 161 | s, sent := replacedFixture(t, st, "restored-away") |
| 162 | if _, err := st.rawBoot.Exec(`ALTER TABLE users RENAME TO users_away`); err != nil { |
| 163 | t.Fatal(err) |
| 164 | } |
| 165 | app := hostops.App{Account: "bab", Name: "demo"} |
| 166 | if err := s.sweepOneClaimAdmission(t.Context(), app); !errors.Is(err, ErrSQLDatabaseReplaced) { |
| 167 | t.Fatalf("sweep: %v", err) |
| 168 | } |
| 169 | if _, known, _ := st.AlertSeen(sqlReceiptAlertBox, sqlReplacedRule("bab", "demo", "old"), noMailboxAddress); known { |
| 170 | t.Fatal("a transient recipient failure was recorded as no mailbox") |
| 171 | } |
| 172 | if _, err := st.rawBoot.Exec(`ALTER TABLE users_away RENAME TO users`); err != nil { |
| 173 | t.Fatal(err) |
| 174 | } |
| 175 | if err := s.sweepOneClaimAdmission(t.Context(), app); !errors.Is(err, ErrSQLDatabaseReplaced) { |
| 176 | t.Fatalf("sweep: %v", err) |
| 177 | } |
| 178 | if len(*sent) != 1 || (*sent)[0] != "ops-a@keymail.dev" { |
| 179 | t.Fatalf("alert not delivered after the recipients recovered: %v", *sent) |
| 180 | } |
| 181 | } |
| 182 | |
| 183 | // m-4: the missing-grant note is logged when an alert is delivered, not on |
| 184 | // every sweep that finds nothing new to say. |
| 185 | func TestPlan9Fix2ReplacedAlertMissingGrantLoggedOnce(t *testing.T) { |
| 186 | st := openGatedStore(t) |
| 187 | s, sent := replacedFixture(t, st, "restored-away") |
| 188 | buf := captureLog(t) |
| 189 | app := hostops.App{Account: "bab", Name: "demo"} |
| 190 | for range 3 { |
| 191 | if err := s.sweepOneClaimAdmission(t.Context(), app); !errors.Is(err, ErrSQLDatabaseReplaced) { |
| 192 | t.Fatalf("sweep: %v", err) |
| 193 | } |
| 194 | } |
| 195 | if len(*sent) != 1 { |
| 196 | t.Fatalf("sent %v, want one delivery", *sent) |
| 197 | } |
| 198 | if n := strings.Count(buf.String(), "no keymail grant for ops-b@keymail.dev"); n != 1 { |
| 199 | t.Fatalf("missing-grant note logged %d times over three sweeps:\n%s", n, buf.String()) |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | // m-4: a database replaced under the running console alerts like a restored |
| 204 | // one, rather than surfacing only as an identity error. |
| 205 | func TestPlan9Fix2DatabaseReplacedUnderRunningConsoleAlerts(t *testing.T) { |
| 206 | st := openGatedStore(t) |
| 207 | id, err := st.sqlDatabaseID() |
| 208 | if err != nil { |
| 209 | t.Fatal(err) |
| 210 | } |
| 211 | s, sent := replacedFixture(t, st, id) |
| 212 | // Replace the file under the running process: same path, new inode. |
| 213 | path := st.gate.path |
| 214 | b, err := os.ReadFile(path) |
| 215 | if err != nil { |
| 216 | t.Fatal(err) |
| 217 | } |
| 218 | if err = os.WriteFile(path+".restored", b, 0o600); err != nil { |
| 219 | t.Fatal(err) |
| 220 | } |
| 221 | if err = os.Rename(path+".restored", path); err != nil { |
| 222 | t.Fatal(err) |
| 223 | } |
| 224 | err = s.sweepOneClaimAdmission(t.Context(), hostops.App{Account: "bab", Name: "demo"}) |
| 225 | if !errors.Is(err, ErrSQLDatabaseReplaced) { |
| 226 | t.Fatalf("live replacement surfaced only as %v", err) |
| 227 | } |
| 228 | if len(*sent) != 1 { |
| 229 | t.Fatalf("no operator alert for a live replacement: %v", *sent) |
| 230 | } |
| 231 | } |
| 232 | |
| 233 | // m-5: a callback that calls back into the outer, captured Store (rather |
| 234 | // than the view it was given) is refused at once instead of waiting on the |
| 235 | // connection the view holds, and no remote check runs first. |
| 236 | func TestPlan9Fix2OuterStoreInsideViewRefused(t *testing.T) { |
| 237 | s := openGatedStore(t) |
| 238 | seedGateApp(t, s) |
| 239 | t.Run("nested mutation through the outer store", func(t *testing.T) { |
| 240 | calls := 0 |
| 241 | g := sqlGrant{scope: gateScope, kind: grantEffect, check: func(context.Context) error { calls++; return nil }} |
| 242 | ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) |
| 243 | defer cancel() |
| 244 | err := s.withClaimSQL(ctx, []claimSQLScope{gateScope}, allowAllClaimSQL, func(*Store) error { |
| 245 | return s.WithAppMutation(ctx, []sqlGrant{g}, func(*Store) error { return nil }) |
| 246 | }) |
| 247 | if !errors.Is(err, ErrClaimSQLScope) { |
| 248 | t.Fatalf("nested mutation through the outer store: %v", err) |
| 249 | } |
| 250 | if calls != 0 { |
| 251 | t.Fatalf("ran %d remote checks before refusing", calls) |
| 252 | } |
| 253 | }) |
| 254 | t.Run("plain outer query", func(t *testing.T) { |
| 255 | ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) |
| 256 | defer cancel() |
| 257 | var qerr error |
| 258 | err := s.withClaimSQL(ctx, []claimSQLScope{gateScope}, allowAllClaimSQL, func(*Store) error { |
| 259 | _, qerr = s.Operators() |
| 260 | return nil |
| 261 | }) |
| 262 | if err != nil { |
| 263 | t.Fatal(err) |
| 264 | } |
| 265 | if !errors.Is(qerr, ErrClaimSQLScope) { |
| 266 | t.Fatalf("outer query inside the view: %v", qerr) |
| 267 | } |
| 268 | }) |
| 269 | // Other goroutines still wait their turn rather than being refused. |
| 270 | t.Run("other goroutine waits", func(t *testing.T) { |
| 271 | release := make(chan struct{}) |
| 272 | held := make(chan struct{}) |
| 273 | done := make(chan error, 1) |
| 274 | go func() { |
| 275 | done <- s.withClaimSQL(t.Context(), []claimSQLScope{gateScope}, allowAllClaimSQL, func(*Store) error { |
| 276 | close(held) |
| 277 | <-release |
| 278 | return nil |
| 279 | }) |
| 280 | }() |
| 281 | <-held |
| 282 | got := make(chan error, 1) |
| 283 | go func() { _, err := s.Operators(); got <- err }() |
| 284 | time.Sleep(50 * time.Millisecond) |
| 285 | close(release) |
| 286 | if err := <-done; err != nil { |
| 287 | t.Fatal(err) |
| 288 | } |
| 289 | if err := <-got; err != nil { |
| 290 | t.Fatalf("concurrent outer call refused: %v", err) |
| 291 | } |
| 292 | }) |
| 293 | } |
| 294 | |
amadan