| 1 | package main |
| 2 | |
| 3 | // 🤖 The live end-to-end hibernation test ("the money test", #550 Part 4 v2 |
| 4 | // Phase 2): a REAL instance process behind the REAL router, replicating to |
| 5 | // REAL S3. It proves the whole loop on one box — wake fresh, write data |
| 6 | // through the public API, idle out (final checkpoint + lease release), DELETE |
| 7 | // the local DB, wake again (restore from S3), and read the data back. |
| 8 | // |
| 9 | // Gated on LCHAT_TEST_S3_BUCKET like the activator package's spike tests, so |
| 10 | // `go test ./...` stays green without AWS credentials. Run it with: |
| 11 | // |
| 12 | // eval "$(aws configure export-credentials --profile eleven-dev --format env)" |
| 13 | // LCHAT_TEST_S3_BUCKET=<bucket> go test -run TestActivatorLiveHibernation -v . |
| 14 | |
| 15 | import ( |
| 16 | "bytes" |
| 17 | "context" |
| 18 | "crypto/rand" |
| 19 | "encoding/hex" |
| 20 | "encoding/json" |
| 21 | "fmt" |
| 22 | "io" |
| 23 | "net" |
| 24 | "net/http" |
| 25 | "net/http/httptest" |
| 26 | "os" |
| 27 | "os/exec" |
| 28 | "path/filepath" |
| 29 | "strings" |
| 30 | "testing" |
| 31 | "time" |
| 32 | |
| 33 | "github.com/aws/aws-sdk-go-v2/aws" |
| 34 | awsconfig "github.com/aws/aws-sdk-go-v2/config" |
| 35 | awss3 "github.com/aws/aws-sdk-go-v2/service/s3" |
| 36 | s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" |
| 37 | |
| 38 | "lchat/internal/activator" |
| 39 | ) |
| 40 | |
| 41 | func TestActivatorLiveHibernation(t *testing.T) { |
| 42 | bucket := os.Getenv("LCHAT_TEST_S3_BUCKET") |
| 43 | if bucket == "" { |
| 44 | t.Skip("LCHAT_TEST_S3_BUCKET not set; skipping live S3 hibernation test") |
| 45 | } |
| 46 | region := os.Getenv("LCHAT_TEST_S3_REGION") |
| 47 | if region == "" { |
| 48 | region = "eu-west-1" |
| 49 | } |
| 50 | rb := make([]byte, 4) |
| 51 | _, _ = rand.Read(rb) |
| 52 | prefix := fmt.Sprintf("activator-live/%d-%s", time.Now().Unix(), hex.EncodeToString(rb)) |
| 53 | t.Cleanup(func() { cleanupS3Prefix(t, bucket, prefix, region) }) |
| 54 | |
| 55 | const host = "hib-live.test" |
| 56 | |
| 57 | // Sandbox the box-layout globals (control.go / versions.go) in temp dirs. |
| 58 | // The socket dir must stay short — sun_path is ~104 bytes on macOS. |
| 59 | dataDir := t.TempDir() |
| 60 | sockDir := fmt.Sprintf("/tmp/lchat-act-%d", os.Getpid()) |
| 61 | if err := os.MkdirAll(sockDir, 0o755); err != nil { |
| 62 | t.Fatal(err) |
| 63 | } |
| 64 | t.Cleanup(func() { os.RemoveAll(sockDir) }) |
| 65 | optTmp := t.TempDir() |
| 66 | oldData, oldSock, oldOpt := instanceDataDir, instanceSocketDir, optDir |
| 67 | instanceDataDir, instanceSocketDir, optDir = dataDir, sockDir, optTmp |
| 68 | t.Cleanup(func() { instanceDataDir, instanceSocketDir, optDir = oldData, oldSock, oldOpt }) |
| 69 | |
| 70 | // The activator execs the same live-symlinked binary a systemd unit would, |
| 71 | // so build the real lchat binary (the test binary must never be spawned). |
| 72 | bin := filepath.Join(optTmp, "lchat-built") |
| 73 | build := exec.Command("go", "build", "-o", bin, ".") |
| 74 | build.Env = append(os.Environ(), "GOTOOLCHAIN=auto") |
| 75 | if out, err := build.CombinedOutput(); err != nil { |
| 76 | t.Fatalf("go build: %v\n%s", err, out) |
| 77 | } |
| 78 | if err := stageVersion(bin, "livetest"); err != nil { |
| 79 | t.Fatalf("stage: %v", err) |
| 80 | } |
| 81 | if err := pointLive(host, "livetest"); err != nil { |
| 82 | t.Fatalf("point live: %v", err) |
| 83 | } |
| 84 | |
| 85 | // The instance's EnvironmentFile, as add-instance would have written it. |
| 86 | adminToken := "livetest-" + hex.EncodeToString(rb) |
| 87 | if err := writeEnvFile(filepath.Join(dataDir, host+".env"), |
| 88 | map[string]string{"LCHAT_ADMIN_TOKEN": adminToken}); err != nil { |
| 89 | t.Fatalf("env file: %v", err) |
| 90 | } |
| 91 | |
| 92 | reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db")) |
| 93 | if err != nil { |
| 94 | t.Fatalf("open registry: %v", err) |
| 95 | } |
| 96 | route := Route{Host: host, Kind: "instance", |
| 97 | Socket: filepath.Join(sockDir, host+".sock"), |
| 98 | DBPath: filepath.Join(dataDir, host+".db"), |
| 99 | // Per-host opt-in: only hibernate-flagged routes are activator-managed. |
| 100 | Hibernate: true} |
| 101 | if err := reg.upsert(route); err != nil { |
| 102 | t.Fatalf("upsert: %v", err) |
| 103 | } |
| 104 | |
| 105 | cfg := hibernateConfig{Bucket: bucket, Region: region, Prefix: prefix, Idle: 3 * time.Second} |
| 106 | act := newActivator(cfg, &s3Backend{cfg: cfg}, reg) |
| 107 | act.start() |
| 108 | t.Cleanup(func() { |
| 109 | ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) |
| 110 | defer cancel() |
| 111 | act.shutdown(ctx) |
| 112 | }) |
| 113 | |
| 114 | rt, err := newRouter(reg, act) |
| 115 | if err != nil { |
| 116 | t.Fatalf("new router: %v", err) |
| 117 | } |
| 118 | front := httptest.NewServer(rt) |
| 119 | t.Cleanup(front.Close) |
| 120 | |
| 121 | get := func(path string) (int, string) { |
| 122 | t.Helper() |
| 123 | req, _ := http.NewRequest("GET", front.URL+path, nil) |
| 124 | req.Host = host |
| 125 | resp, err := http.DefaultClient.Do(req) |
| 126 | if err != nil { |
| 127 | t.Fatalf("GET %s: %v", path, err) |
| 128 | } |
| 129 | defer resp.Body.Close() |
| 130 | body, _ := io.ReadAll(resp.Body) |
| 131 | return resp.StatusCode, string(body) |
| 132 | } |
| 133 | |
| 134 | // --- Act 1: first wake is a FRESH boot (nothing in S3, no local DB). --- |
| 135 | start := time.Now() |
| 136 | if code, _ := get("/api/group"); code != 200 { |
| 137 | t.Fatalf("first wake: GET /api/group = %d, want 200", code) |
| 138 | } |
| 139 | t.Logf("wake #1 (fresh boot, no replica) served in %s", time.Since(start).Truncate(time.Millisecond)) |
| 140 | |
| 141 | // Write real data through the public path: the admin names the space. |
| 142 | groupName := "Hibernation survives " + hex.EncodeToString(rb) |
| 143 | body, _ := json.Marshal(map[string]string{"name": groupName}) |
| 144 | req, _ := http.NewRequest("POST", front.URL+"/api/admin/group", bytes.NewReader(body)) |
| 145 | req.Host = host |
| 146 | req.Header.Set("Authorization", "Bearer "+adminToken) |
| 147 | req.Header.Set("Content-Type", "application/json") |
| 148 | resp, err := http.DefaultClient.Do(req) |
| 149 | if err != nil { |
| 150 | t.Fatalf("set group name: %v", err) |
| 151 | } |
| 152 | if resp.StatusCode != 200 { |
| 153 | b, _ := io.ReadAll(resp.Body) |
| 154 | resp.Body.Close() |
| 155 | t.Fatalf("set group name: %d %s", resp.StatusCode, b) |
| 156 | } |
| 157 | resp.Body.Close() |
| 158 | t.Logf("wrote group name %q through the admin API", groupName) |
| 159 | |
| 160 | // The due-time probe answers on the real instance (admin-gated; nothing |
| 161 | // scheduled → 0). The sleep below re-runs the same call over the socket. |
| 162 | nreq, _ := http.NewRequest("GET", front.URL+"/api/next-due", nil) |
| 163 | nreq.Host = host |
| 164 | nreq.Header.Set("Authorization", "Bearer "+adminToken) |
| 165 | nresp, err := http.DefaultClient.Do(nreq) |
| 166 | if err != nil { |
| 167 | t.Fatalf("GET /api/next-due: %v", err) |
| 168 | } |
| 169 | nbody, _ := io.ReadAll(nresp.Body) |
| 170 | nresp.Body.Close() |
| 171 | if nresp.StatusCode != 200 || !strings.Contains(string(nbody), `"due":0`) { |
| 172 | t.Fatalf("next-due probe: %d %s, want 200 with due 0", nresp.StatusCode, nbody) |
| 173 | } |
| 174 | |
| 175 | // --- Act 2: idle out. Final checkpoint to S3, lease released, process gone. --- |
| 176 | waitForLive(t, "hibernation after idle", 90*time.Second, func() bool { return act.entryCount() == 0 }) |
| 177 | // The sleep recorded the (empty) due-time index in the registry. |
| 178 | if rt, err := reg.get(host); err != nil || rt == nil || rt.NextDue != 0 { |
| 179 | t.Fatalf("registry next_due after sleep = %+v, %v; want 0", rt, err) |
| 180 | } |
| 181 | if _, err := net.DialTimeout("unix", route.Socket, 300*time.Millisecond); err == nil { |
| 182 | t.Fatal("instance socket still accepting after hibernation") |
| 183 | } |
| 184 | if _, err := os.Stat(route.DBPath); err != nil { |
| 185 | t.Fatalf("local DB should remain as a warm cache after sleep: %v", err) |
| 186 | } |
| 187 | t.Logf("instance hibernated (idle %s); local DB left as warm cache", cfg.Idle) |
| 188 | |
| 189 | // --- Act 3: destroy the local state; the wake path must restore from S3. --- |
| 190 | matches, _ := filepath.Glob(route.DBPath + "*") |
| 191 | for _, m := range matches { |
| 192 | if err := os.RemoveAll(m); err != nil { |
| 193 | t.Fatalf("remove %s: %v", m, err) |
| 194 | } |
| 195 | } |
| 196 | t.Logf("deleted local state: %v", matches) |
| 197 | |
| 198 | start = time.Now() |
| 199 | code, groupJSON := get("/api/group") |
| 200 | if code != 200 { |
| 201 | t.Fatalf("wake from S3: GET /api/group = %d, want 200", code) |
| 202 | } |
| 203 | if !strings.Contains(groupJSON, groupName) { |
| 204 | t.Fatalf("restored instance lost the data: %s does not contain %q", groupJSON, groupName) |
| 205 | } |
| 206 | t.Logf("wake #2 (restored from s3://%s/%s) served in %s; data survived: %s", |
| 207 | bucket, prefix, time.Since(start).Truncate(time.Millisecond), groupJSON) |
| 208 | |
| 209 | // --- Act 4: idle out again and prove the lease was released cleanly. --- |
| 210 | waitForLive(t, "second hibernation", 90*time.Second, func() bool { return act.entryCount() == 0 }) |
| 211 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
| 212 | defer cancel() |
| 213 | held, err := activator.AcquireLease(ctx, bucket, cfg.hibernatePrefix(host), region) |
| 214 | if err != nil { |
| 215 | t.Fatalf("lease not cleanly released after sleep: %v", err) |
| 216 | } |
| 217 | if err := held.Release(ctx); err != nil { |
| 218 | t.Fatalf("release probe lease: %v", err) |
| 219 | } |
| 220 | t.Log("second sleep released the lease cleanly (probe acquire+release succeeded)") |
| 221 | } |
| 222 | |
| 223 | // waitForLive is waitFor with a caller-chosen deadline — S3 round trips make |
| 224 | // the live test's transitions slower than the fakes'. |
| 225 | func waitForLive(t *testing.T, what string, timeout time.Duration, cond func() bool) { |
| 226 | t.Helper() |
| 227 | deadline := time.Now().Add(timeout) |
| 228 | for time.Now().Before(deadline) { |
| 229 | if cond() { |
| 230 | return |
| 231 | } |
| 232 | time.Sleep(100 * time.Millisecond) |
| 233 | } |
| 234 | t.Fatalf("timed out waiting for %s", what) |
| 235 | } |
| 236 | |
| 237 | // cleanupS3Prefix deletes every object AND version under the test prefix so |
| 238 | // live runs leave nothing behind (mirrors the activator package's helper). |
| 239 | func cleanupS3Prefix(t *testing.T, bucket, prefix, region string) { |
| 240 | t.Helper() |
| 241 | ctx := context.Background() |
| 242 | cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) |
| 243 | if err != nil { |
| 244 | t.Logf("cleanup aws config: %v", err) |
| 245 | return |
| 246 | } |
| 247 | client := awss3.NewFromConfig(cfg) |
| 248 | p := awss3.NewListObjectVersionsPaginator(client, &awss3.ListObjectVersionsInput{ |
| 249 | Bucket: aws.String(bucket), |
| 250 | Prefix: aws.String(prefix), |
| 251 | }) |
| 252 | for p.HasMorePages() { |
| 253 | out, err := p.NextPage(ctx) |
| 254 | if err != nil { |
| 255 | t.Logf("cleanup list versions: %v", err) |
| 256 | return |
| 257 | } |
| 258 | var objs []s3types.ObjectIdentifier |
| 259 | for _, v := range out.Versions { |
| 260 | objs = append(objs, s3types.ObjectIdentifier{Key: v.Key, VersionId: v.VersionId}) |
| 261 | } |
| 262 | for _, m := range out.DeleteMarkers { |
| 263 | objs = append(objs, s3types.ObjectIdentifier{Key: m.Key, VersionId: m.VersionId}) |
| 264 | } |
| 265 | if len(objs) == 0 { |
| 266 | continue |
| 267 | } |
| 268 | if _, err := client.DeleteObjects(ctx, &awss3.DeleteObjectsInput{ |
| 269 | Bucket: aws.String(bucket), |
| 270 | Delete: &s3types.Delete{Objects: objs}, |
| 271 | }); err != nil { |
| 272 | t.Logf("cleanup delete: %v", err) |
| 273 | } |
| 274 | } |
| 275 | } |
| 276 | |
amadan