eleven / ocho Public

Clone
git clone https://amadan.net/eleven/ocho

Plain git — no account needed to clone.

Download

Download this file

1package main
2
3// πŸ€– `lchat relocate <host> --to <region>` β€” move a space to another region
4// while keeping its STABLE ADDRESS (multi-edge Phase 1, docs/multi-edge.md).
5//
6// A space's address is its identity (invite links, per-origin keys bind to it),
7// so it must never change; its region is a mutable property (`home_region`), so
8// it must be free to move. Relocation is the operator/admin action that changes
9// where the one copy lives without changing its name. Three moving parts, in the
10// order they run:
11//
12// 1. Store copy. The destination region's box wakes the space by RESTORING its
13// DB from that region's object store, so the replica must exist there first.
14// On the dev mesh both regions share ONE bucket, so the bytes are already
15// visible β€” the copy is a NO-OP (relocationStorePlan detects same
16// bucket+prefix+endpoint β†’ skip). In production each region has its own
17// store, so it is a real prefix-to-prefix copy of s3://<bucket>/<host>/…
18// (copyS3Prefix), done while the space sleeps.
19// 2. Flip home_region + re-fan-out. setHomeRegion writes the replicated routes
20// column; fanOutMeshRoute pushes the updated route to peers so every box's
21// candidatesFor (mesh.go) re-homes the space onto a box in the destination
22// region. The local router needs no reload β€” its mesh reads home_region live
23// from the shared registry each placement decision.
24// 3. (Optional) A-record move. Repointing <host>'s A record at the destination
25// box avoids the proxy hop, but is NOT required for correctness: the
26// *.<domain> wildcard + the mesh's proxy-to-owner already route any request
27// to the new owner through the DNS-TTL window. Off by default; --move-dns
28// opts in, and only Route53 implements it today (route53dns.go).
29//
30// WHY THIS IS DATA-SAFE β€” the single-writer lease. Relocation never
31// split-brains because the S3 lease (internal/activator: lock.json under the
32// space's prefix, acquired via a conditional write) is the ONE truth of who may
33// write, and home_region/DNS are only hints that steer placement:
34//
35// - Shared bucket (dev): source and destination contend on the SAME lock.json,
36// so the lease is globally singular. After the flip the destination becomes
37// the HRW owner; if the source is momentarily still awake it holds the lease,
38// the destination's AcquireLease fails loudly (LeaseExistsError β†’ 503 +
39// Retry-After), and the client's retry lands once the source sleeps and
40// releases. Never two writers, ever.
41// - Per-region buckets (prod): the copy must run while the source is ASLEEP
42// (lease released, replica fully flushed by the sleep-time SyncAndWait), so
43// the destination restores a complete copy and acquires a FRESH lease in its
44// own bucket. Once home_region is flipped the source's owns() returns false,
45// so it never wakes there again β€” a stray request proxies to the new owner
46// instead. copyS3Prefix deliberately skips lock.json so a copied, live lease
47// can't stall the destination's first wake.
48//
49// IDEMPOTENCY + the copy-clobber guard. Re-running a completed relocation is a
50// no-op: once home_region already equals the target the space's authoritative
51// copy is the DESTINATION's, so relocate must NOT re-copy source→dest (that
52// would overwrite newer destination writes with stale source bytes). So the
53// copy + flip run ONLY on the forward move (current != target); an
54// already-home re-run just heals peers by re-fanning-out the route. A crash
55// between copy and flip converges safely: the source is still authoritative
56// (not yet flipped), so the next run re-copies (idempotent overwrite) and flips.
57
58import (
59 "context"
60 "flag"
61 "fmt"
62 "log"
63 "os"
64 "path"
65 "strings"
66
67 "github.com/aws/aws-sdk-go-v2/aws"
68 awsconfig "github.com/aws/aws-sdk-go-v2/config"
69 "github.com/aws/aws-sdk-go-v2/credentials"
70 awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
71 "path/filepath"
72)
73
74// storeLocation names one region's object store for a space's replica prefix.
75// Endpoint distinguishes same-named buckets across accounts/providers (AWS S3
76// vs an R2 endpoint) so the skip test in relocationStorePlan is never fooled by
77// a bucket-name coincidence.
78type storeLocation struct {
79 Bucket string
80 Prefix string // the per-host key prefix, e.g. "<hibernate-prefix>/<host>"
81 Endpoint string // "" = AWS default endpoint; set for R2 / custom endpoints
82}
83
84// storePlan is relocationStorePlan's decision: whether the cross-region store
85// copy is a no-op (the dev shared bucket β€” the bytes are already there) or a
86// real prefix-to-prefix copy (prod's per-region stores).
87type storePlan struct {
88 Skip bool
89 Reason string
90 Src storeLocation
91 Dst storeLocation
92}
93
94// relocationStorePlan is the PURE decision at the heart of the store-copy step,
95// split out so it is unit-testable with no S3. It returns skip=true exactly when
96// the source and destination name the same bytes β€” same bucket, same per-host
97// prefix, and same endpoint β€” which is the dev mesh's one-shared-bucket case:
98// both regions already read the same replica, so there is nothing to copy.
99// Anything else is a real copy plan the caller executes with copyS3Prefix.
100func relocationStorePlan(src, dst storeLocation) storePlan {
101 if src.Bucket == dst.Bucket && src.Prefix == dst.Prefix && src.Endpoint == dst.Endpoint {
102 return storePlan{
103 Skip: true,
104 Reason: "source and destination share the same bucket, prefix and endpoint β€” the replica bytes are already visible to the destination region (the dev shared-bucket case)",
105 Src: src, Dst: dst,
106 }
107 }
108 return storePlan{
109 Skip: false,
110 Reason: fmt.Sprintf("cross-region prefix copy: s3://%s/%s/ β†’ s3://%s/%s/", src.Bucket, src.Prefix, dst.Bucket, dst.Prefix),
111 Src: src, Dst: dst,
112 }
113}
114
115// s3CopyAPI is the slice of the S3 client copyS3Prefix needs β€” an interface so
116// the copy is unit-tested against a fake without reaching AWS (the same shape
117// s3blobstore.go and route53dns.go use for their clients).
118type s3CopyAPI interface {
119 ListObjectsV2(ctx context.Context, in *awss3.ListObjectsV2Input, optFns ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error)
120 CopyObject(ctx context.Context, in *awss3.CopyObjectInput, optFns ...func(*awss3.Options)) (*awss3.CopyObjectOutput, error)
121}
122
123// copyS3Prefix copies every object under src's per-host prefix to dst's,
124// preserving the relative key layout, via server-side CopyObject (no bytes
125// transit this process). It EXCLUDES the lease object (lock.json): a copied,
126// still-valid lease would make the destination's first AcquireLease see a held
127// lock and 503 until the TTL expires β€” the destination must start with a clean
128// lease and take its own. Returns the count copied.
129//
130// Same-account, same-credential copy is the realistic prod shape (regional
131// buckets in one AWS account; CopyObject works cross-region). Cross-ACCOUNT or
132// cross-ENDPOINT (e.g. an R2 destination) needs a download+reupload variant β€”
133// a documented follow-on (docs/multi-edge.md), not this MVP.
134func copyS3Prefix(ctx context.Context, client s3CopyAPI, srcBucket, srcPrefix, dstBucket, dstPrefix string) (int, error) {
135 srcPrefix = strings.Trim(srcPrefix, "/")
136 dstPrefix = strings.Trim(dstPrefix, "/")
137 listPrefix := srcPrefix + "/"
138 p := awss3.NewListObjectsV2Paginator(client, &awss3.ListObjectsV2Input{
139 Bucket: aws.String(srcBucket),
140 Prefix: aws.String(listPrefix),
141 })
142 copied := 0
143 for p.HasMorePages() {
144 page, err := p.NextPage(ctx)
145 if err != nil {
146 return copied, fmt.Errorf("relocate: list s3://%s/%s: %w", srcBucket, listPrefix, err)
147 }
148 for _, obj := range page.Contents {
149 srcKey := aws.ToString(obj.Key)
150 rel := strings.TrimPrefix(srcKey, listPrefix)
151 if rel == "" || path.Base(rel) == "lock.json" {
152 continue // never copy the lease β€” the destination must take a fresh one
153 }
154 dstKey := dstPrefix + "/" + rel
155 // CopySource is "<bucket>/<key>"; our keys are ASCII-safe (host FQDN
156 // prefixes + litestream LTX / base64url leaves), so no URL-escaping
157 // is needed. A cross-account/cross-endpoint copier would encode here.
158 copySource := srcBucket + "/" + srcKey
159 if _, err := client.CopyObject(ctx, &awss3.CopyObjectInput{
160 Bucket: aws.String(dstBucket),
161 Key: aws.String(dstKey),
162 CopySource: aws.String(copySource),
163 }); err != nil {
164 return copied, fmt.Errorf("relocate: copy %s β†’ s3://%s/%s: %w", copySource, dstBucket, dstKey, err)
165 }
166 copied++
167 }
168 }
169 return copied, nil
170}
171
172// aRecordUpserter is the OPTIONAL A-record half of relocation β€” a sibling to the
173// dnsProvider seam (wildcard.go), kept separate so the ACME-TXT path stays
174// untouched and a provider can support DNS-01 without supporting A moves.
175// route53DNS implements it (route53dns.go); DNSimple does not yet, so --move-dns
176// refuses on DNSimple with a clear message. Relocation is correct WITHOUT it
177// (the wildcard + proxy route to the new owner), so this is never on the MVP's
178// critical path.
179type aRecordUpserter interface {
180 upsertA(ctx context.Context, zone, host, ipv4 string) error
181}
182
183// relocateOpts parameterizes one relocation. Reusable so a future controld RPC
184// or member-facing "change region" button can call relocate() with the same
185// core (the CLI runRelocate is one caller).
186type relocateOpts struct {
187 Host string
188 ToRegion string
189 // Dst names the destination region's object store for the cross-region copy.
190 // The zero value means "the same store as the source" β€” the dev shared-bucket
191 // case, where the copy is a no-op. Prefix defaults to the source's per-host
192 // prefix when empty.
193 Dst storeLocation
194 // MoveDNS opts into repointing <host>'s A record at DNSTarget (off by
195 // default). DNSZone overrides the hosted zone; empty derives it from the
196 // wildcard/mesh domains the host falls under.
197 MoveDNS bool
198 DNSTarget string
199 DNSZone string
200}
201
202// relocateResult reports what happened, for the CLI to print.
203type relocateResult struct {
204 Host string
205 FromRegion string
206 ToRegion string
207 Plan storePlan
208 Copied int // objects copied (0 on the skip path)
209 AlreadyHome bool // home_region already == ToRegion β€” a converging no-op
210 DNSMoved bool
211}
212
213// relocate is the shared relocation core behind `lchat relocate` (and any future
214// RPC/button). It validates, does the store copy (or skips it), flips
215// home_region, re-fans-out the route, and optionally moves the A record. mesh
216// answers the "is the destination region live?" precondition and is beaten once
217// here for a fresh view; the actual route replication reuses fanOutMeshRoute.
218func relocate(reg *Registry, mesh *meshSvc, hc *hibernateConfig, opts relocateOpts) (*relocateResult, error) {
219 host := strings.ToLower(strings.TrimSpace(opts.Host))
220 if !validHost(host) {
221 return nil, fmt.Errorf("%w: %q", errInvalidHost, host)
222 }
223 region := strings.TrimSpace(opts.ToRegion)
224 if region == "" {
225 return nil, fmt.Errorf("relocate: --to <region> is required (clearing a space's region back to agnostic is a separate operation, not relocate)")
226 }
227 if hc == nil {
228 return nil, fmt.Errorf("relocate: hibernation is not configured on this box (LCHAT_HIBERNATE_BUCKET unset) β€” relocation is hibernation-backed: the space's state must live in S3 for the destination region to restore it")
229 }
230 if mesh == nil {
231 return nil, fmt.Errorf("relocate: the activator mesh is not configured (LCHAT_MESH_SELF/_DOMAINS/_TOKEN/_REGION) β€” placement-by-region needs it; run on a mesh box with /data/router.env sourced")
232 }
233
234 rt, err := reg.get(host)
235 if err != nil {
236 return nil, err
237 }
238 if rt == nil {
239 return nil, fmt.Errorf("%w: %q (run relocate on a box whose registry has the route β€” typically the source region's box)", errUnknownHost, host)
240 }
241 if rt.Kind != "instance" {
242 return nil, fmt.Errorf("relocate: %q is a %s route, not an instance", host, rt.Kind)
243 }
244 if rt.Status != "ready" {
245 return nil, fmt.Errorf("relocate: %s is %s, not ready β€” fix the instance first", host, rt.Status)
246 }
247 if !rt.Hibernate {
248 return nil, fmt.Errorf("relocate: %s is systemd-managed, not hibernating β€” a live systemd instance is pinned to this box; run `lchat hibernate %s --on` first so its state lives in S3 and can be restored in the destination region", host, host)
249 }
250 if !mesh.inScope(host) {
251 return nil, fmt.Errorf("relocate: %s is not under a configured mesh domain (%s) β€” only in-scope hosts are placed by region", host, strings.Join(mesh.domains, ", "))
252 }
253
254 res := &relocateResult{Host: host, FromRegion: rt.HomeRegion, ToRegion: region}
255
256 // Idempotent + copy-clobber guard: once the space is already homed to the
257 // target, the DESTINATION holds the authoritative copy. Re-copying source→dest
258 // would clobber newer destination writes, so DON'T. Just heal any peer that
259 // missed the earlier flip by re-fanning-out the route, and return.
260 // The instance env rides every fan-out (meshRoutePayload doc) so the new
261 // owner wakes this space with its real admin token, not a token-less boot.
262 instEnv, _ := readEnvFile(filepath.Join(instanceDataDir, host+".env"))
263
264 if rt.HomeRegion == region {
265 res.AlreadyHome = true
266 here := storeLocation{Bucket: hc.Bucket, Prefix: hc.hibernatePrefix(host)}
267 res.Plan = relocationStorePlan(here, here) // same store β†’ Skip
268 fanOutMeshRoute(hc, *rt, instEnv) // best-effort heal; a no-op when the mesh is off
269 return res, nil
270 }
271
272 // The destination region must have a live box, or the flip re-homes the space
273 // onto nothing (candidatesFor would just fall back to the whole fleet and the
274 // space would not actually move). beat() first, exactly like fanOutMeshRoute,
275 // so the membership + peer-region view is fresh.
276 mesh.beat()
277 if !mesh.regionLive(region) {
278 return nil, fmt.Errorf("relocate: no live mesh box advertises region %q β€” bring up a box with LCHAT_MESH_REGION=%s (and confirm this box has its own LCHAT_MESH_REGION set so it can read peers' regions)", region, region)
279 }
280
281 // Step 1 β€” the store copy (or the dev no-op).
282 src := storeLocation{Bucket: hc.Bucket, Prefix: hc.hibernatePrefix(host)}
283 dst := opts.Dst
284 if dst.Bucket == "" {
285 dst = src // no destination store named β†’ same store (dev shared bucket)
286 }
287 if dst.Prefix == "" {
288 dst.Prefix = hc.hibernatePrefix(host)
289 }
290 res.Plan = relocationStorePlan(src, dst)
291 if !res.Plan.Skip {
292 copied, err := runStoreCopy(context.Background(), hc.Region, res.Plan)
293 if err != nil {
294 return nil, err
295 }
296 res.Copied = copied
297 }
298
299 // Step 2 β€” flip home_region and re-fan-out the updated route to peers.
300 if err := reg.setHomeRegion(host, region); err != nil {
301 return nil, fmt.Errorf("relocate: set home_region: %w", err)
302 }
303 updated := *rt
304 updated.HomeRegion = region
305 fanOutMeshRoute(hc, updated, instEnv)
306
307 // Step 3 β€” optional A-record move.
308 if opts.MoveDNS {
309 if err := moveARecord(context.Background(), mesh, host, opts); err != nil {
310 // The flip already succeeded, so relocation is correct via the proxy;
311 // surface the DNS failure but don't unwind the (correct) move.
312 return res, fmt.Errorf("relocate: home_region flipped to %s, but the optional A-record move failed: %w (relocation is still correct β€” the wildcard + proxy route to the new owner)", region, err)
313 }
314 res.DNSMoved = true
315 }
316 return res, nil
317}
318
319// runStoreCopy builds one S3 client and runs copyS3Prefix. Only reached on the
320// cross-bucket (prod) path β€” the dev shared-bucket case skips before here. Uses
321// the source region as the signing region (a same-account client copies
322// cross-region via CopyObject regardless) and, when the destination names a
323// custom endpoint (e.g. R2), addresses it path-style. Optional static
324// credentials for the destination store ride LCHAT_RELOCATE_DST_ACCESS_KEY /
325// _SECRET_KEY, mirroring s3blobstore.go's mirror-store pattern; unset = the
326// default AWS chain.
327func runStoreCopy(ctx context.Context, srcRegion string, plan storePlan) (int, error) {
328 var opts []func(*awsconfig.LoadOptions) error
329 if srcRegion != "" {
330 opts = append(opts, awsconfig.WithRegion(srcRegion))
331 }
332 if ak := strings.TrimSpace(os.Getenv("LCHAT_RELOCATE_DST_ACCESS_KEY")); ak != "" {
333 opts = append(opts, awsconfig.WithCredentialsProvider(
334 credentials.NewStaticCredentialsProvider(ak, strings.TrimSpace(os.Getenv("LCHAT_RELOCATE_DST_SECRET_KEY")), "")))
335 }
336 cfg, err := awsconfig.LoadDefaultConfig(ctx, opts...)
337 if err != nil {
338 return 0, fmt.Errorf("relocate: load aws config for copy: %w", err)
339 }
340 client := awss3.NewFromConfig(cfg, func(o *awss3.Options) {
341 if plan.Dst.Endpoint != "" {
342 o.BaseEndpoint = aws.String(plan.Dst.Endpoint)
343 o.UsePathStyle = true
344 }
345 })
346 log.Printf("relocate: %s", plan.Reason)
347 return copyS3Prefix(ctx, client, plan.Src.Bucket, plan.Src.Prefix, plan.Dst.Bucket, plan.Dst.Prefix)
348}
349
350// moveARecord repoints host's A record via whichever DNS provider is configured,
351// if it supports A upserts. Off unless the caller passed --move-dns.
352func moveARecord(ctx context.Context, mesh *meshSvc, host string, opts relocateOpts) error {
353 if strings.TrimSpace(opts.DNSTarget) == "" {
354 return fmt.Errorf("--move-dns needs --dns-target <ipv4> (the destination box's address)")
355 }
356 prov, reason := dnsProviderFromEnv()
357 if prov == nil {
358 return fmt.Errorf("no DNS provider configured: %s", reason)
359 }
360 up, ok := prov.(aRecordUpserter)
361 if !ok {
362 return fmt.Errorf("the configured DNS provider can't move A records yet (only LCHAT_DNS_PROVIDER=route53 does) β€” omit --move-dns: relocation is correct without it (the wildcard + proxy route to the new owner)")
363 }
364 zone := strings.TrimSpace(opts.DNSZone)
365 if zone == "" {
366 zone = zoneForHost(host, mesh.domains)
367 }
368 if zone == "" {
369 return fmt.Errorf("could not derive a hosted zone for %s from the mesh domains β€” pass --dns-zone", host)
370 }
371 return up.upsertA(ctx, zone, host, strings.TrimSpace(opts.DNSTarget))
372}
373
374// zoneForHost picks the hosted zone for host from the configured domains: the
375// longest domain that host is a subdomain of (so a delegated sub-zone wins over
376// its parent). "" when none matches.
377func zoneForHost(host string, domains []string) string {
378 host = strings.ToLower(strings.TrimSuffix(host, "."))
379 best := ""
380 for _, d := range domains {
381 d = strings.ToLower(strings.TrimSpace(d))
382 if d == "" {
383 continue
384 }
385 if strings.HasSuffix(host, "."+d) && len(d) > len(best) {
386 best = d
387 }
388 }
389 return best
390}
391
392// runRelocate: `lchat relocate <host> --to <region> [flags]`.
393func runRelocate(args []string) {
394 fs := flag.NewFlagSet("relocate", flag.ExitOnError)
395 registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")
396 to := fs.String("to", "", "destination region (matches a live box's LCHAT_MESH_REGION) β€” required")
397 dstBucket := fs.String("dst-bucket", "", "destination region's object-store bucket (prod per-region stores); empty = same store as source (the dev shared-bucket no-op)")
398 dstPrefix := fs.String("dst-prefix", "", "destination per-host key prefix; empty = the same prefix the source uses")
399 dstEndpoint := fs.String("dst-endpoint", "", "destination S3 endpoint (e.g. an R2 URL); empty = AWS default")
400 moveDNS := fs.Bool("move-dns", false, "also repoint the host's A record at the destination box (OPTIONAL β€” off by default; relocation is correct without it, the wildcard + proxy route to the new owner)")
401 dnsTarget := fs.String("dns-target", "", "destination box IPv4 for the A record (required with --move-dns)")
402 dnsZone := fs.String("dns-zone", "", "hosted zone for the A record; empty = derived from the mesh domains")
403 _ = fs.Parse(reorderFlagsFirst(fs, args))
404
405 host := strings.ToLower(strings.TrimSpace(fs.Arg(0)))
406 if !validHost(host) {
407 die("relocate: invalid host %q (usage: lchat relocate <host> --to <region>)", host)
408 }
409 region := strings.TrimSpace(*to)
410 if region == "" {
411 die("relocate: --to <region> is required")
412 }
413
414 hc, err := hibernateConfigFromEnv()
415 if err != nil {
416 die("relocate: %v", err)
417 }
418 mesh, err := meshFromEnv(hc)
419 if err != nil {
420 die("relocate: mesh config: %v", err)
421 }
422
423 reg, err := openRegistry(*registryPath)
424 if err != nil {
425 die("relocate: open registry: %v", err)
426 }
427 if mesh != nil {
428 mesh.reg = reg // so any region read resolves against the local registry
429 }
430
431 res, err := relocate(reg, mesh, hc, relocateOpts{
432 Host: host,
433 ToRegion: region,
434 Dst: storeLocation{
435 Bucket: strings.TrimSpace(*dstBucket),
436 Prefix: strings.TrimSpace(*dstPrefix),
437 Endpoint: strings.TrimSpace(*dstEndpoint),
438 },
439 MoveDNS: *moveDNS,
440 DNSTarget: strings.TrimSpace(*dnsTarget),
441 DNSZone: strings.TrimSpace(*dnsZone),
442 })
443 if err != nil {
444 die("relocate: %v", err)
445 }
446
447 if res.AlreadyHome {
448 fmt.Printf("%s is already homed to %s β€” re-fanned the route out to heal any peer that missed it (no copy, no re-flip)\n", res.Host, res.ToRegion)
449 return
450 }
451 from := res.FromRegion
452 if from == "" {
453 from = "(agnostic)"
454 }
455 fmt.Printf("relocated %s: %s β†’ %s\n", res.Host, from, res.ToRegion)
456 if res.Plan.Skip {
457 fmt.Printf(" store: no copy β€” %s\n", res.Plan.Reason)
458 } else {
459 fmt.Printf(" store: copied %d object(s) β€” %s\n", res.Copied, res.Plan.Reason)
460 }
461 fmt.Printf(" home_region flipped + route re-fanned-out to peers β€” the mesh now places %s on a %s box (the single-writer lease keeps one writer through the move)\n", res.Host, res.ToRegion)
462 if res.DNSMoved {
463 fmt.Printf(" A record moved to the destination box (%s)\n", strings.TrimSpace(*dnsTarget))
464 } else {
465 fmt.Printf(" A record unchanged β€” the wildcard + mesh proxy route to the new owner through the DNS-TTL window (use --move-dns --dns-target to avoid the proxy hop)\n")
466 }
467}
468