| 1 | package main |
| 2 | |
| 3 | // Tests for relocate.go — the multi-edge Phase 1 "relocate" action. Three |
| 4 | // concerns, matching the load-bearing decisions: |
| 5 | // |
| 6 | // - relocationStorePlan: the PURE skip-vs-copy decision (dev's shared bucket |
| 7 | // is a no-op; anything differing is a real copy). |
| 8 | // - copyS3Prefix: the cross-region prefix copy — key layout preserved, the |
| 9 | // lease (lock.json) EXCLUDED, tested against a fake S3 client. |
| 10 | // - relocate: target-region validation, the home_region flip, and the |
| 11 | // idempotent + copy-clobber guard (an already-home re-run never re-copies). |
| 12 | |
| 13 | import ( |
| 14 | "context" |
| 15 | "path/filepath" |
| 16 | "strings" |
| 17 | "testing" |
| 18 | "time" |
| 19 | |
| 20 | "github.com/aws/aws-sdk-go-v2/aws" |
| 21 | awss3 "github.com/aws/aws-sdk-go-v2/service/s3" |
| 22 | s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" |
| 23 | ) |
| 24 | |
| 25 | // --- relocationStorePlan (pure) ------------------------------------------------ |
| 26 | |
| 27 | func TestRelocationStorePlan(t *testing.T) { |
| 28 | base := storeLocation{Bucket: "shared", Prefix: "hib/space1.dev.11ch.at", Endpoint: ""} |
| 29 | |
| 30 | // Same bucket + prefix + endpoint = the dev shared-bucket case: the bytes are |
| 31 | // already visible to both regions, so the copy is a no-op. |
| 32 | if p := relocationStorePlan(base, base); !p.Skip { |
| 33 | t.Fatalf("identical src/dst should skip, got copy: %+v", p) |
| 34 | } |
| 35 | |
| 36 | // A different bucket is a real copy (prod's per-region stores). |
| 37 | dstBucket := base |
| 38 | dstBucket.Bucket = "fi-bucket" |
| 39 | if p := relocationStorePlan(base, dstBucket); p.Skip { |
| 40 | t.Fatal("different bucket must NOT skip — it needs a real copy") |
| 41 | } |
| 42 | |
| 43 | // A different prefix is a real copy. |
| 44 | dstPrefix := base |
| 45 | dstPrefix.Prefix = "hib2/space1.dev.11ch.at" |
| 46 | if p := relocationStorePlan(base, dstPrefix); p.Skip { |
| 47 | t.Fatal("different prefix must NOT skip") |
| 48 | } |
| 49 | |
| 50 | // A same-named bucket at a DIFFERENT endpoint (S3 vs R2) is different bytes — |
| 51 | // the endpoint must be part of the identity test. |
| 52 | dstEndpoint := base |
| 53 | dstEndpoint.Endpoint = "https://acct.r2.cloudflarestorage.com" |
| 54 | if p := relocationStorePlan(base, dstEndpoint); p.Skip { |
| 55 | t.Fatal("different endpoint must NOT skip — a bucket-name coincidence is not the same bytes") |
| 56 | } |
| 57 | } |
| 58 | |
| 59 | // --- copyS3Prefix -------------------------------------------------------------- |
| 60 | |
| 61 | // fakeS3Copy is an in-memory stand-in for the S3 client copyS3Prefix drives: |
| 62 | // ListObjectsV2 returns the seeded keys under the requested prefix (one page), |
| 63 | // CopyObject records every call. |
| 64 | type fakeS3Copy struct { |
| 65 | objects []string |
| 66 | copies []*awss3.CopyObjectInput |
| 67 | } |
| 68 | |
| 69 | func (f *fakeS3Copy) ListObjectsV2(_ context.Context, in *awss3.ListObjectsV2Input, _ ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) { |
| 70 | prefix := aws.ToString(in.Prefix) |
| 71 | var contents []s3types.Object |
| 72 | for _, k := range f.objects { |
| 73 | if strings.HasPrefix(k, prefix) { |
| 74 | contents = append(contents, s3types.Object{Key: aws.String(k)}) |
| 75 | } |
| 76 | } |
| 77 | return &awss3.ListObjectsV2Output{Contents: contents}, nil |
| 78 | } |
| 79 | |
| 80 | func (f *fakeS3Copy) CopyObject(_ context.Context, in *awss3.CopyObjectInput, _ ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error) { |
| 81 | f.copies = append(f.copies, in) |
| 82 | return &awss3.CopyObjectOutput{}, nil |
| 83 | } |
| 84 | |
| 85 | func TestCopyS3Prefix(t *testing.T) { |
| 86 | src := "hib/space1.dev.11ch.at" |
| 87 | dst := "hib/space1.dev.11ch.at" // same per-host prefix, different bucket |
| 88 | fake := &fakeS3Copy{objects: []string{ |
| 89 | src + "/lock.json", // the lease — MUST be skipped |
| 90 | src + "/ltx/0001.ltx", // replica data |
| 91 | src + "/snapshots/000.snap", // replica data |
| 92 | "other/space2.dev.11ch.at/ltx/0001.ltx", // a different host — out of prefix |
| 93 | }} |
| 94 | |
| 95 | n, err := copyS3Prefix(context.Background(), fake, "src-bucket", src, "fi-bucket", dst) |
| 96 | if err != nil { |
| 97 | t.Fatalf("copyS3Prefix: %v", err) |
| 98 | } |
| 99 | if n != 2 { |
| 100 | t.Fatalf("copied %d objects, want 2 (lock.json excluded, other host out of prefix)", n) |
| 101 | } |
| 102 | if len(fake.copies) != 2 { |
| 103 | t.Fatalf("recorded %d CopyObject calls, want 2", len(fake.copies)) |
| 104 | } |
| 105 | got := map[string]string{} // dst key -> CopySource |
| 106 | for _, c := range fake.copies { |
| 107 | if aws.ToString(c.Bucket) != "fi-bucket" { |
| 108 | t.Fatalf("copy dst bucket = %q, want fi-bucket", aws.ToString(c.Bucket)) |
| 109 | } |
| 110 | got[aws.ToString(c.Key)] = aws.ToString(c.CopySource) |
| 111 | } |
| 112 | // The lease was never copied. |
| 113 | for k := range got { |
| 114 | if strings.HasSuffix(k, "lock.json") { |
| 115 | t.Fatalf("lock.json was copied (%q) — a copied lease stalls the destination's first wake", k) |
| 116 | } |
| 117 | } |
| 118 | // Keys map into the destination prefix, CopySource points at the source. |
| 119 | if cs, ok := got["hib/space1.dev.11ch.at/ltx/0001.ltx"]; !ok || cs != "src-bucket/hib/space1.dev.11ch.at/ltx/0001.ltx" { |
| 120 | t.Fatalf("ltx object: dstKey/CopySource wrong (%q -> %q)", "hib/space1.dev.11ch.at/ltx/0001.ltx", cs) |
| 121 | } |
| 122 | if _, ok := got["hib/space1.dev.11ch.at/snapshots/000.snap"]; !ok { |
| 123 | t.Fatalf("snapshot object was not copied to the right dst key; got %v", got) |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | // --- relocate: validation + flip + idempotency --------------------------------- |
| 128 | |
| 129 | // relocTestReg seeds an in-memory registry with one hibernating, ready, |
| 130 | // in-scope instance route homed to fromRegion. |
| 131 | func relocTestReg(t *testing.T, host, fromRegion string, hibernate bool, kind string) *Registry { |
| 132 | t.Helper() |
| 133 | reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db")) |
| 134 | if err != nil { |
| 135 | t.Fatalf("open registry: %v", err) |
| 136 | } |
| 137 | if err := reg.upsert(Route{ |
| 138 | Host: host, Kind: kind, Status: "ready", |
| 139 | Hibernate: hibernate, HomeRegion: fromRegion, |
| 140 | }); err != nil { |
| 141 | t.Fatalf("seed route: %v", err) |
| 142 | } |
| 143 | return reg |
| 144 | } |
| 145 | |
| 146 | // relocTestMesh builds a mesh over a fake heartbeat store: this box is |
| 147 | // selfRegion, and each peer in peers advertises its region. relocate.beat() |
| 148 | // refreshes membership + regions off this store. |
| 149 | func relocTestMesh(reg *Registry, self, selfRegion string, peers map[string]string) *meshSvc { |
| 150 | now := time.Now() |
| 151 | store := newFakeMeshStore(func() time.Time { return now }) |
| 152 | for peer, region := range peers { |
| 153 | store.setBeat(peer, now) |
| 154 | store.setRegion(peer, region) |
| 155 | } |
| 156 | m := newMesh(self, store, []string{"dev.11ch.at"}) |
| 157 | m.region = selfRegion |
| 158 | m.reg = reg |
| 159 | m.now = func() time.Time { return now } |
| 160 | return m |
| 161 | } |
| 162 | |
| 163 | func TestRelocateFlipAndIdempotency(t *testing.T) { |
| 164 | host := "space1.dev.11ch.at" |
| 165 | reg := relocTestReg(t, host, "ie", true, "instance") |
| 166 | mesh := relocTestMesh(reg, "https://ie-box", "ie", map[string]string{"https://fi-box": "fi"}) |
| 167 | hc := &hibernateConfig{Bucket: "shared", Prefix: "hib"} // one shared bucket → skip path |
| 168 | |
| 169 | // Forward move ie → fi: flips home_region, skip copy (shared bucket). |
| 170 | res, err := relocate(reg, mesh, hc, relocateOpts{Host: host, ToRegion: "fi"}) |
| 171 | if err != nil { |
| 172 | t.Fatalf("relocate: %v", err) |
| 173 | } |
| 174 | if res.AlreadyHome { |
| 175 | t.Fatal("first move should not be AlreadyHome") |
| 176 | } |
| 177 | if !res.Plan.Skip || res.Copied != 0 { |
| 178 | t.Fatalf("shared bucket should skip the copy, got plan=%+v copied=%d", res.Plan, res.Copied) |
| 179 | } |
| 180 | if rt, _ := reg.get(host); rt == nil || rt.HomeRegion != "fi" { |
| 181 | t.Fatalf("home_region not flipped to fi: %+v", rt) |
| 182 | } |
| 183 | |
| 184 | // Re-run to the same region: AlreadyHome no-op, home_region unchanged, and — |
| 185 | // critically — NO copy even though we now point at a DIFFERENT bucket (the |
| 186 | // copy-clobber guard: the destination is authoritative once home, so |
| 187 | // re-copying stale source bytes would lose newer writes). |
| 188 | res2, err := relocate(reg, mesh, hc, relocateOpts{ |
| 189 | Host: host, ToRegion: "fi", |
| 190 | Dst: storeLocation{Bucket: "some-other-bucket"}, // would force a copy if reached |
| 191 | }) |
| 192 | if err != nil { |
| 193 | t.Fatalf("idempotent relocate: %v", err) |
| 194 | } |
| 195 | if !res2.AlreadyHome { |
| 196 | t.Fatal("re-run to the same region must be AlreadyHome") |
| 197 | } |
| 198 | if res2.Copied != 0 || !res2.Plan.Skip { |
| 199 | t.Fatalf("already-home re-run must NOT copy (copy-clobber guard), got copied=%d plan=%+v", res2.Copied, res2.Plan) |
| 200 | } |
| 201 | if rt, _ := reg.get(host); rt.HomeRegion != "fi" { |
| 202 | t.Fatalf("home_region changed on idempotent re-run: %q", rt.HomeRegion) |
| 203 | } |
| 204 | } |
| 205 | |
| 206 | func TestRelocateValidation(t *testing.T) { |
| 207 | host := "space1.dev.11ch.at" |
| 208 | hc := &hibernateConfig{Bucket: "shared", Prefix: "hib"} |
| 209 | liveFi := map[string]string{"https://fi-box": "fi"} |
| 210 | |
| 211 | cases := []struct { |
| 212 | name string |
| 213 | reg *Registry |
| 214 | mesh *meshSvc |
| 215 | hc *hibernateConfig |
| 216 | opts relocateOpts |
| 217 | wantSub string |
| 218 | }{ |
| 219 | { |
| 220 | name: "missing target region", |
| 221 | reg: relocTestReg(t, host, "ie", true, "instance"), |
| 222 | hc: hc, |
| 223 | opts: relocateOpts{Host: host, ToRegion: ""}, |
| 224 | wantSub: "--to <region> is required", |
| 225 | }, |
| 226 | { |
| 227 | name: "hibernation not configured", |
| 228 | reg: relocTestReg(t, host, "ie", true, "instance"), |
| 229 | hc: nil, |
| 230 | opts: relocateOpts{Host: host, ToRegion: "fi"}, |
| 231 | wantSub: "hibernation is not configured", |
| 232 | }, |
| 233 | { |
| 234 | name: "mesh not configured", |
| 235 | reg: relocTestReg(t, host, "ie", true, "instance"), |
| 236 | hc: hc, |
| 237 | mesh: nil, |
| 238 | opts: relocateOpts{Host: host, ToRegion: "fi"}, |
| 239 | wantSub: "activator mesh is not configured", |
| 240 | }, |
| 241 | { |
| 242 | name: "unknown host", |
| 243 | reg: relocTestReg(t, host, "ie", true, "instance"), |
| 244 | hc: hc, |
| 245 | opts: relocateOpts{Host: "nope.dev.11ch.at", ToRegion: "fi"}, |
| 246 | wantSub: "unknown host", |
| 247 | }, |
| 248 | { |
| 249 | name: "not an instance", |
| 250 | reg: relocTestReg(t, host, "ie", true, "service"), |
| 251 | hc: hc, |
| 252 | opts: relocateOpts{Host: host, ToRegion: "fi"}, |
| 253 | wantSub: "not an instance", |
| 254 | }, |
| 255 | { |
| 256 | name: "not hibernating (box-pinned)", |
| 257 | reg: relocTestReg(t, host, "ie", false, "instance"), |
| 258 | hc: hc, |
| 259 | opts: relocateOpts{Host: host, ToRegion: "fi"}, |
| 260 | wantSub: "systemd-managed", |
| 261 | }, |
| 262 | { |
| 263 | name: "no live box in target region", |
| 264 | reg: relocTestReg(t, host, "ie", true, "instance"), |
| 265 | hc: hc, |
| 266 | opts: relocateOpts{Host: host, ToRegion: "zz"}, |
| 267 | wantSub: "no live mesh box advertises region", |
| 268 | }, |
| 269 | } |
| 270 | |
| 271 | for _, tc := range cases { |
| 272 | t.Run(tc.name, func(t *testing.T) { |
| 273 | mesh := tc.mesh |
| 274 | // A nil mesh in the case means "test the nil-mesh guard"; otherwise |
| 275 | // build a live-fi mesh. The "mesh not configured" case sets mesh:nil |
| 276 | // explicitly and we honour it. |
| 277 | if mesh == nil && tc.name != "mesh not configured" { |
| 278 | mesh = relocTestMesh(tc.reg, "https://ie-box", "ie", liveFi) |
| 279 | } |
| 280 | _, err := relocate(tc.reg, mesh, tc.hc, tc.opts) |
| 281 | if err == nil { |
| 282 | t.Fatalf("want error containing %q, got nil", tc.wantSub) |
| 283 | } |
| 284 | if !strings.Contains(err.Error(), tc.wantSub) { |
| 285 | t.Fatalf("error %q does not contain %q", err.Error(), tc.wantSub) |
| 286 | } |
| 287 | }) |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | // zoneForHost picks the longest matching configured domain (a delegated |
| 292 | // sub-zone wins over its parent). |
| 293 | func TestZoneForHost(t *testing.T) { |
| 294 | domains := []string{"11ch.at", "dev.11ch.at"} |
| 295 | if z := zoneForHost("space1.dev.11ch.at", domains); z != "dev.11ch.at" { |
| 296 | t.Fatalf("zone = %q, want dev.11ch.at (most specific)", z) |
| 297 | } |
| 298 | if z := zoneForHost("chat.11ch.at", domains); z != "11ch.at" { |
| 299 | t.Fatalf("zone = %q, want 11ch.at", z) |
| 300 | } |
| 301 | if z := zoneForHost("space1.example.com", domains); z != "" { |
| 302 | t.Fatalf("zone = %q, want empty (no configured domain matches)", z) |
| 303 | } |
| 304 | } |
| 305 | |
amadan