| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "log" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/mail" |
| 14 | "github.com/carlosframework/platform/internal/release" |
| 15 | ) |
| 16 | |
| 17 | // Spec §9: expire -> trash -> purge at 3 days, on a fake clock. |
| 18 | func TestExpiryPassExpiresThenPurgesAtThreeDays(t *testing.T) { |
| 19 | fx := newConsoleFixture(t) |
| 20 | ctx := context.Background() |
| 21 | now := time.Now().Truncate(time.Second).Add(time.Hour) |
| 22 | tempApp(t, fx, "tmp-a", now.Add(-time.Minute)) |
| 23 | fx.srv.expiryPass(ctx, now) |
| 24 | tomb := markerOf(t, fx, "tmp-a") |
| 25 | if tomb == nil || !tomb.DeletedAt.Equal(now) || tomb.Actor != expiryActor { |
| 26 | t.Fatalf("marker = %+v, want one written by the pass at %v", tomb, now) |
| 27 | } |
| 28 | if l := lifeOf(t, fx, "tmp-a"); l.DeletedAt != now.Unix() || l.PurgeAfter != now.Unix()+72*3600 { |
| 29 | t.Fatalf("row = %+v, want trashed with the 3-day window", l) |
| 30 | } |
| 31 | fx.srv.expiryPass(ctx, now.Add(72*time.Hour-time.Second)) |
| 32 | if markerOf(t, fx, "tmp-a") == nil { |
| 33 | t.Fatal("purged before the 3 days were up") |
| 34 | } |
| 35 | fx.srv.expiryPass(ctx, now.Add(72*time.Hour)) |
| 36 | if keys, _ := fx.rel.List(ctx, "apps/"+fx.sqid+"/tmp-a/"); len(keys) != 0 { |
| 37 | t.Fatalf("objects left after the early purge: %v", keys) |
| 38 | } |
| 39 | if l := lifeOf(t, fx, "tmp-a"); l != nil { |
| 40 | t.Fatalf("row = %+v, want it gone with the name", l) |
| 41 | } |
| 42 | } |
| 43 | |
| 44 | // Review Focus 3. |
| 45 | func TestExpiryBoundaryIsInclusive(t *testing.T) { |
| 46 | fx := newConsoleFixture(t) |
| 47 | now := time.Now().Truncate(time.Second) |
| 48 | tempApp(t, fx, "tmp-now", now) |
| 49 | tempApp(t, fx, "tmp-later", now.Add(time.Second)) |
| 50 | fx.srv.expiryPass(context.Background(), now) |
| 51 | if l := lifeOf(t, fx, "tmp-now"); !l.Trashed() { |
| 52 | t.Error("expires_at == now did not expire") |
| 53 | } |
| 54 | if l := lifeOf(t, fx, "tmp-later"); l.Trashed() { |
| 55 | t.Error("expires_at == now+1s expired early") |
| 56 | } |
| 57 | } |
| 58 | |
| 59 | // Spec §9: a keep between candidate listing and lock acquisition; the pass |
| 60 | // must skip. |
| 61 | func TestExpirySkipsAKeepThatLandedAfterTheListing(t *testing.T) { |
| 62 | fx := newConsoleFixture(t) |
| 63 | now := time.Now().Truncate(time.Second) |
| 64 | tempApp(t, fx, "tmp-a", now.Add(-time.Minute)) |
| 65 | rows, _ := fx.st.TemporaryAppRows() |
| 66 | fx.st.KeepApp(OpsAccountID, "tmp-a") |
| 67 | for _, r := range rows { |
| 68 | fx.srv.expireOne(context.Background(), r, now, always(true)) |
| 69 | } |
| 70 | if markerOf(t, fx, "tmp-a") != nil { |
| 71 | t.Fatal("the pass trashed an app a keep had made permanent") |
| 72 | } |
| 73 | if l := lifeOf(t, fx, "tmp-a"); l.Trashed() || l.ExpiresAt != 0 { |
| 74 | t.Fatalf("row = %+v, want live and permanent", l) |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | // Spec §9: failure at step 3 retries. |
| 79 | func TestExpiryRetriesAFailedTrash(t *testing.T) { |
| 80 | fx := newConsoleFixture(t) |
| 81 | now := time.Now().Truncate(time.Second) |
| 82 | tempApp(t, fx, "tmp-a", now.Add(-time.Minute)) |
| 83 | fx.srv.cfg.Releases = &hookStore{Store: fx.rel, failPut: map[string]int{release.AppTombstoneKey(fx.sqid, "tmp-a"): 1}} |
| 84 | fx.srv.expiryPass(context.Background(), now) |
| 85 | if markerOf(t, fx, "tmp-a") != nil || lifeOf(t, fx, "tmp-a").Trashed() { |
| 86 | t.Fatal("a failed marker write changed something") |
| 87 | } |
| 88 | fx.srv.expiryPass(context.Background(), now.Add(5*time.Minute)) |
| 89 | if l := lifeOf(t, fx, "tmp-a"); !l.Trashed() || l.DeletedAt != now.Add(5*time.Minute).Unix() { |
| 90 | t.Fatalf("row after the retry = %+v", l) |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | // Spec §9: restore -> keep -> trash between decision and purge; the stale |
| 95 | // decision refuses through PurgeAppIfTrashedAt. |
| 96 | func TestEarlyPurgeRefusesAMarkerFromAnotherTrash(t *testing.T) { |
| 97 | fx := newConsoleFixture(t) |
| 98 | ctx := context.Background() |
| 99 | now := time.Now().Truncate(time.Second) |
| 100 | decided := now.Add(-72 * time.Hour) |
| 101 | trashedTemp(t, fx, "tmp-a", decided) |
| 102 | // The bucket moved on: restored, kept and trashed again by hand. |
| 103 | release.RestoreApp(ctx, fx.rel, fx.sqid, "tmp-a") |
| 104 | release.TrashApp(ctx, fx.rel, fx.sqid, "tmp-a", "paul@keymail.dev", now.Add(-time.Hour)) |
| 105 | rows, _ := fx.st.PurgeDueApps(now) |
| 106 | for _, r := range rows { |
| 107 | fx.srv.purgeOne(ctx, r, now, always(true)) |
| 108 | } |
| 109 | if keys, _ := fx.rel.List(ctx, "apps/"+fx.sqid+"/tmp-a/"); len(keys) < 2 { |
| 110 | t.Fatalf("a stale decision destroyed objects: %v", keys) |
| 111 | } |
| 112 | if lifeOf(t, fx, "tmp-a") == nil { |
| 113 | t.Fatal("a stale decision freed the name") |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | // Spec §9: an interrupted purge resumes. |
| 118 | func TestEarlyPurgeResumesAfterAnInterruption(t *testing.T) { |
| 119 | fx := newConsoleFixture(t) |
| 120 | ctx := context.Background() |
| 121 | now := time.Now().Truncate(time.Second) |
| 122 | trashedTemp(t, fx, "tmp-a", now.Add(-72*time.Hour)) |
| 123 | fx.rel.Put(ctx, "apps/"+fx.sqid+"/tmp-a/releases/v1/site.tar.gz", []byte("x"), "") |
| 124 | fx.srv.cfg.Releases = &hookStore{Store: fx.rel, failDelete: map[string]int{"apps/" + fx.sqid + "/tmp-a/releases/v1/site.tar.gz": 1}} |
| 125 | fx.srv.expiryPass(ctx, now) |
| 126 | if lifeOf(t, fx, "tmp-a") == nil || markerOf(t, fx, "tmp-a") == nil { |
| 127 | t.Fatal("an interrupted purge lost its marker or its row") |
| 128 | } |
| 129 | fx.srv.expiryPass(ctx, now.Add(5*time.Minute)) |
| 130 | if l := lifeOf(t, fx, "tmp-a"); l != nil { |
| 131 | t.Fatalf("row = %+v after the resumed purge, want gone", l) |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | // Spec §9: ErrNotTrashed with an empty prefix finalises; with objects, restores. |
| 136 | // |
| 137 | // The branch is only reached when the marker disappears AFTER purgeOne's |
| 138 | // settle read it and BEFORE PurgeAppIfTrashedAt reads it: a marker gone |
| 139 | // before the pass is repaired by the settle sweep and never gets that far. |
| 140 | // So the hook deletes the marker on its second read. Reads of the marker key |
| 141 | // in one pass over a due, consistent row: purgeOne's settleApp (1), then |
| 142 | // PurgeAppIfTrashedAt's purgeReady (2). The sweep reads nothing for a row |
| 143 | // whose trash state already agrees with the listing. |
| 144 | func TestEarlyPurgeNotTrashedBranches(t *testing.T) { |
| 145 | for _, objects := range []bool{false, true} { |
| 146 | t.Run(map[bool]string{false: "no objects finalises", true: "objects restore"}[objects], func(t *testing.T) { |
| 147 | fx := newConsoleFixture(t) |
| 148 | ctx := context.Background() |
| 149 | now := time.Now().Truncate(time.Second) |
| 150 | trashedTemp(t, fx, "tmp-a", now.Add(-72*time.Hour)) |
| 151 | if !objects { |
| 152 | // The marker is the only key left under the prefix. kass's |
| 153 | // objects keep the tripwire satisfied. |
| 154 | if err := fx.rel.Delete(ctx, "apps/"+fx.sqid+"/tmp-a/channels/edge.json"); err != nil { |
| 155 | t.Fatal(err) |
| 156 | } |
| 157 | } |
| 158 | marker := release.AppTombstoneKey(fx.sqid, "tmp-a") |
| 159 | reads := 0 |
| 160 | fx.srv.cfg.Releases = &hookStore{Store: fx.rel, onGet: func(key string) { |
| 161 | if key != marker { |
| 162 | return |
| 163 | } |
| 164 | reads++ |
| 165 | if reads == 2 { |
| 166 | fx.rel.Delete(ctx, marker) |
| 167 | } |
| 168 | }} |
| 169 | var logged bytes.Buffer |
| 170 | log.SetOutput(&logged) |
| 171 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 172 | |
| 173 | fx.srv.expiryPass(ctx, now) |
| 174 | |
| 175 | if reads < 2 { |
| 176 | t.Fatalf("the marker was read %d time(s); the purge never reached its own precondition read", reads) |
| 177 | } |
| 178 | if !strings.Contains(logged.String(), "left the trash before its purge") { |
| 179 | t.Fatalf("the ErrNotTrashed branch did not run; log:\n%s", logged.String()) |
| 180 | } |
| 181 | l := lifeOf(t, fx, "tmp-a") |
| 182 | if !objects { |
| 183 | if l != nil { |
| 184 | t.Fatalf("row = %+v, want finalised (no marker, no objects, evidence)", l) |
| 185 | } |
| 186 | return |
| 187 | } |
| 188 | if l == nil || l.Trashed() || l.Restoring || l.PurgeAfter != 0 || l.ExpiresAt != now.Add(temporaryTTLDefault).Unix() { |
| 189 | t.Fatalf("row = %+v, want restored: live, no deadline, expiry now+7d", l) |
| 190 | } |
| 191 | if keys, _ := fx.rel.List(ctx, "apps/"+fx.sqid+"/tmp-a/"); len(keys) == 0 { |
| 192 | t.Fatal("the restore branch destroyed objects") |
| 193 | } |
| 194 | }) |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | // Spec §9: control-plane and email-estate backstops refuse without tearing down. |
| 199 | func TestEarlyPurgeBackstops(t *testing.T) { |
| 200 | ctx := context.Background() |
| 201 | now := time.Now().Truncate(time.Second) |
| 202 | t.Run("control plane", func(t *testing.T) { |
| 203 | fx := newConsoleFixture(t) |
| 204 | trashedTemp(t, fx, "tmp-a", now.Add(-72*time.Hour)) |
| 205 | // Unreachable through any door (Task 6); set by hand to prove the backstop. |
| 206 | fx.st.db.Exec(`UPDATE apps SET control_plane = 1 WHERE app = 'tmp-a'`) |
| 207 | fx.srv.expiryPass(ctx, now) |
| 208 | if markerOf(t, fx, "tmp-a") == nil || lifeOf(t, fx, "tmp-a") == nil { |
| 209 | t.Fatal("the pass purged the control plane") |
| 210 | } |
| 211 | }) |
| 212 | t.Run("email estate", func(t *testing.T) { |
| 213 | fx := newConsoleFixture(t) |
| 214 | trashedTemp(t, fx, "tmp-a", now.Add(-72*time.Hour)) |
| 215 | putMailRecord(t, fx, "tmp-a", &mail.Record{ |
| 216 | Declared: &mail.Declared{EnvPrefix: "CARLOS_SMTP", From: "hi@example.com", At: now}, |
| 217 | Provisioned: &mail.Provisioned{KeyID: "AKIA1", Active: true, At: now}, |
| 218 | }) |
| 219 | fx.srv.expiryPass(ctx, now) |
| 220 | if markerOf(t, fx, "tmp-a") == nil || lifeOf(t, fx, "tmp-a") == nil { |
| 221 | t.Fatal("the pass purged an app with email estate") |
| 222 | } |
| 223 | }) |
| 224 | } |
| 225 | |
| 226 | // Spec §9 "Empty bucket": startup materialize and several passes against an |
| 227 | // empty replacement bucket, then one holding only trash markers. No marker is |
| 228 | // written and both rows survive unchanged. |
| 229 | func TestExpiryPassAgainstTheWrongBucket(t *testing.T) { |
| 230 | for _, markersOnly := range []bool{false, true} { |
| 231 | fx := newConsoleFixture(t) |
| 232 | ctx := context.Background() |
| 233 | now := time.Now().Truncate(time.Second) |
| 234 | trashedTemp(t, fx, "tmp-due", now.Add(-72*time.Hour)) |
| 235 | tempApp(t, fx, "tmp-expired", now.Add(-time.Hour)) |
| 236 | before := map[string]AppLife{"tmp-due": *lifeOf(t, fx, "tmp-due"), "tmp-expired": *lifeOf(t, fx, "tmp-expired")} |
| 237 | dir := filepath.Join(t.TempDir(), "wrong") |
| 238 | wrong := &deletableStore{Store: release.NewDirStore(dir), root: dir} |
| 239 | if markersOnly { |
| 240 | release.TrashApp(ctx, wrong, fx.sqid, "someone-else", "paul@keymail.dev", now) |
| 241 | } |
| 242 | fx.srv.cfg.Releases = wrong |
| 243 | materializeQuietly(ctx, wrong, fx.st) |
| 244 | for i := 0; i < 3; i++ { |
| 245 | fx.srv.expiryPass(ctx, now.Add(time.Duration(i)*5*time.Minute)) |
| 246 | } |
| 247 | keys, _ := wrong.List(ctx, "apps/") |
| 248 | if want := map[bool]int{false: 0, true: 1}[markersOnly]; len(keys) != want { |
| 249 | t.Fatalf("markersOnly=%v: the wrong bucket now holds %v", markersOnly, keys) |
| 250 | } |
| 251 | for app, b := range before { |
| 252 | if l := lifeOf(t, fx, app); l == nil || *l != b { |
| 253 | t.Fatalf("markersOnly=%v: %s = %+v, want unchanged %+v", markersOnly, app, l, b) |
| 254 | } |
| 255 | } |
| 256 | } |
| 257 | } |
| 258 | |
| 259 | // Spec §9 "Settle sweep": a manual trash of a temporary app with 30 days left |
| 260 | // whose row update failed. No restart, no user action: the next pass settles |
| 261 | // it, and the early purge runs at 3 days. |
| 262 | func TestSettleSweepRepairsAFailedManualTrash(t *testing.T) { |
| 263 | fx := newConsoleFixture(t) |
| 264 | ctx := context.Background() |
| 265 | now := time.Now().Truncate(time.Second) |
| 266 | tempApp(t, fx, "tmp-a", now.Add(30*24*time.Hour)) |
| 267 | release.TrashApp(ctx, fx.rel, fx.sqid, "tmp-a", "paul@keymail.dev", now) |
| 268 | fx.srv.expiryPass(ctx, now.Add(time.Minute)) |
| 269 | if l := lifeOf(t, fx, "tmp-a"); l.DeletedAt != now.Unix() || l.PurgeAfter != now.Unix()+72*3600 { |
| 270 | t.Fatalf("row after the sweep = %+v, want trashed at the marker's clock with the 3-day window", l) |
| 271 | } |
| 272 | fx.srv.expiryPass(ctx, now.Add(72*time.Hour)) |
| 273 | if l := lifeOf(t, fx, "tmp-a"); l != nil { |
| 274 | t.Fatalf("row = %+v, want purged at 3 days", l) |
| 275 | } |
| 276 | } |
| 277 | |
| 278 | // Spec §9: an early purge whose final marker delete fails, with no other keys |
| 279 | // in the bucket. The next pass trips the tripwire, still retries the purge, |
| 280 | // finishes, and removes the row. |
| 281 | func TestEarlyPurgeOfTheLastAppRetriesUnderTheTripwire(t *testing.T) { |
| 282 | fx := newConsoleFixture(t) |
| 283 | ctx := context.Background() |
| 284 | now := time.Now().Truncate(time.Second) |
| 285 | dir := filepath.Join(t.TempDir(), "only") |
| 286 | only := &deletableStore{Store: release.NewDirStore(dir), root: dir} |
| 287 | fx.srv.cfg.Releases = only |
| 288 | fx.rel = only |
| 289 | trashedTemp(t, fx, "tmp-a", now.Add(-72*time.Hour)) |
| 290 | marker := release.AppTombstoneKey(fx.sqid, "tmp-a") |
| 291 | fx.srv.cfg.Releases = &hookStore{Store: only, failDelete: map[string]int{marker: 1}} |
| 292 | fx.srv.expiryPass(ctx, now) |
| 293 | if keys, _ := only.List(ctx, "apps/"); len(keys) != 1 || keys[0] != marker { |
| 294 | t.Fatalf("after the first pass the bucket holds %v, want only the marker", keys) |
| 295 | } |
| 296 | if lifeOf(t, fx, "tmp-a") == nil { |
| 297 | t.Fatal("the row went before the marker") |
| 298 | } |
| 299 | fx.srv.expiryPass(ctx, now.Add(5*time.Minute)) |
| 300 | if keys, _ := only.List(ctx, "apps/"); len(keys) != 0 { |
| 301 | t.Fatalf("the retry left %v", keys) |
| 302 | } |
| 303 | if l := lifeOf(t, fx, "tmp-a"); l != nil { |
| 304 | t.Fatalf("row = %+v, want gone", l) |
| 305 | } |
| 306 | } |
| 307 | |
| 308 | // Task 7 review: a restore whose marker delete landed and whose row update |
| 309 | // failed is finished by the settle sweep, not by the door. A door retry after |
| 310 | // that answers "not in the trash" and audits nothing, so the sweep must say |
| 311 | // what it did, and bump the serial so hosts bring the routes back promptly. |
| 312 | func TestSettleSweepReportsACatchUpRestore(t *testing.T) { |
| 313 | fx := newConsoleFixture(t) |
| 314 | ctx := context.Background() |
| 315 | now := time.Now().Truncate(time.Second) |
| 316 | trashedTemp(t, fx, "tmp-a", now.Add(-time.Hour)) |
| 317 | if ok, err := fx.st.SetRestoring(OpsAccountID, "tmp-a", true); err != nil || !ok { |
| 318 | t.Fatal(ok, err) |
| 319 | } |
| 320 | if err := release.RestoreApp(ctx, fx.rel, fx.sqid, "tmp-a"); err != nil { |
| 321 | t.Fatal(err) |
| 322 | } |
| 323 | serialBefore, _, _ := fx.rel.Get(ctx, release.SerialKey) |
| 324 | var logged bytes.Buffer |
| 325 | log.SetOutput(&logged) |
| 326 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 327 | |
| 328 | fx.srv.expiryPass(ctx, now) |
| 329 | |
| 330 | if l := lifeOf(t, fx, "tmp-a"); l == nil || l.Trashed() || l.Restoring { |
| 331 | t.Fatalf("row = %+v, want the restore finished", l) |
| 332 | } |
| 333 | if !strings.Contains(logged.String(), "expiry pass: finished a restore of "+fx.sqid+"/tmp-a") { |
| 334 | t.Fatalf("the catch-up restore was silent; log:\n%s", logged.String()) |
| 335 | } |
| 336 | if serialAfter, _, _ := fx.rel.Get(ctx, release.SerialKey); string(serialAfter) == string(serialBefore) { |
| 337 | t.Fatal("the catch-up restore did not bump the serial") |
| 338 | } |
| 339 | } |
| 340 | |
amadan