| 1 | package lease |
| 2 | |
| 3 | // CARLOS_S3_ENDPOINT support for the hibernation lease. |
| 4 | // |
| 5 | // release.NewS3Store has honoured CARLOS_S3_ENDPOINT since the first |
| 6 | // S3-compatible deployment; this package did not, and every S3 client it |
| 7 | // builds dialled real AWS regardless. That gap is not cosmetic: an activator |
| 8 | // route is minted with Hibernate:true, the activator only arms when |
| 9 | // CARLOS_HIBERNATE_BUCKET is set, and arming means acquiring the lease here. |
| 10 | // On a non-AWS store (Hetzner Object Storage, R2, MinIO) the acquire could |
| 11 | // never succeed, so the activator stayed nil and an exec-backed route with |
| 12 | // nothing to spawn it 502'd forever — which took `carlos instances create`, |
| 13 | // the whole console-driven provisioning path, down with it. The same clients |
| 14 | // back the agent's status-publisher election (internal/host/agent.go), so the |
| 15 | // override has to reach all three construction sites, not just the one. |
| 16 | |
| 17 | import ( |
| 18 | "context" |
| 19 | "errors" |
| 20 | "fmt" |
| 21 | "os" |
| 22 | |
| 23 | "github.com/aws/aws-sdk-go-v2/aws" |
| 24 | awsconfig "github.com/aws/aws-sdk-go-v2/config" |
| 25 | awss3 "github.com/aws/aws-sdk-go-v2/service/s3" |
| 26 | lss3 "github.com/benbjohnson/litestream/s3" |
| 27 | "github.com/carlosframework/platform/internal/rootaws" |
| 28 | |
| 29 | "github.com/carlosframework/platform/internal/s3compat" |
| 30 | ) |
| 31 | |
| 32 | // s3Endpoint reports the CARLOS_S3_ENDPOINT override, or "" for stock AWS. |
| 33 | // Delegated to s3compat so there is exactly one reader of that variable. |
| 34 | func s3Endpoint() string { |
| 35 | return s3compat.Endpoint() |
| 36 | } |
| 37 | |
| 38 | // resolveRegion decides the region every client in this package signs for, in |
| 39 | // strict order of authority: |
| 40 | // |
| 41 | // 1. the caller's explicit region (CARLOS_HIBERNATE_REGION), if any; |
| 42 | // 2. whatever the AWS default chain yields — AWS_REGION, or a profile. This |
| 43 | // step is why the resolution cannot be a pure function of the argument: a |
| 44 | // compatible-store deployment sets AWS_REGION to its own location (the |
| 45 | // private CARLOS uses nbg1), and that is the region release.NewS3Store |
| 46 | // already signs for against the same bucket. Substituting a constant here |
| 47 | // would make the lease sign differently from every other S3 client in the |
| 48 | // process, against a store that may well check; |
| 49 | // 3. only if BOTH are empty and an endpoint override is in play, litestream's |
| 50 | // non-AWS default — better a working guess than a client signing for no |
| 51 | // region at all. |
| 52 | // |
| 53 | // Step 3 is the one an override adds. Without it the AWS chain is authoritative |
| 54 | // and an empty result is fine: stock AWS resolves the region per-request from |
| 55 | // the bucket's own endpoint, which a compatible store cannot do. |
| 56 | // |
| 57 | // The resolved value is returned, not just applied, because it has to travel: |
| 58 | // litestream's replica client does its OWN region defaulting (Init substitutes |
| 59 | // the same non-AWS constant for an empty Region), so handing it the resolved |
| 60 | // region is what stops the two halves of one lease — lock.json and the LTX |
| 61 | // files beside it under the same prefix — from signing differently. |
| 62 | func resolveRegion(ctx context.Context, region string) (string, error) { |
| 63 | if region != "" { |
| 64 | return region, nil // no config load needed: step 1 already decided |
| 65 | } |
| 66 | cfg, err := rootaws.LoadDefaultConfig(ctx) |
| 67 | if err != nil { |
| 68 | return "", fmt.Errorf("load aws config: %w", err) |
| 69 | } |
| 70 | return regionFor(region, cfg.Region), nil |
| 71 | } |
| 72 | |
| 73 | // regionFor is resolveRegion's decision with the chain lookup already done — |
| 74 | // the shared rule, so the leaser's path (which loads a config anyway, and must |
| 75 | // not load a second one just to re-ask the same question) and Restore's cannot |
| 76 | // drift apart. |
| 77 | func regionFor(explicit, chainResolved string) string { |
| 78 | if explicit != "" { |
| 79 | return explicit |
| 80 | } |
| 81 | if chainResolved == "" && s3Endpoint() != "" { |
| 82 | return lss3.DefaultRegion |
| 83 | } |
| 84 | return chainResolved |
| 85 | } |
| 86 | |
| 87 | // newLeaserClient builds the S3 client the litestream leaser writes lock.json |
| 88 | // with. Unlike the replica client, which builds its own from its Endpoint |
| 89 | // field, the leaser takes a ready-made client (lss3.Leaser.SetClient) — so the |
| 90 | // override has to be applied right here or the lock lands on AWS while the |
| 91 | // data it guards lands on the compatible store. |
| 92 | // |
| 93 | // The override, path-style addressing, when-required checksums and every |
| 94 | // per-provider quirk come from internal/s3compat, which is a no-op on stock |
| 95 | // AWS — so the flagship's path stays byte-identical while this client picks up |
| 96 | // the unquoted-If-Match fix that litestream's leaser needs to renew a lease on |
| 97 | // Ceph RGW. Passing that client to lss3.Leaser.SetClient is the ONLY way the |
| 98 | // fix reaches litestream, which has no provider handling in its leaser at all. |
| 99 | // |
| 100 | // It returns the region it resolved alongside the client, so AcquireTTL can |
| 101 | // carry that exact value onto the Held for Replicate's replica client rather |
| 102 | // than letting litestream substitute a different default of its own. |
| 103 | func newLeaserClient(ctx context.Context, region string) (*awss3.Client, string, error) { |
| 104 | var opts []func(*awsconfig.LoadOptions) error |
| 105 | if region != "" { |
| 106 | opts = append(opts, awsconfig.WithRegion(region)) |
| 107 | } |
| 108 | cfg, err := rootaws.LoadDefaultConfig(ctx, opts...) |
| 109 | if err != nil { |
| 110 | return nil, "", fmt.Errorf("load aws config: %w", err) |
| 111 | } |
| 112 | cfg.Region = regionFor(region, cfg.Region) |
| 113 | return awss3.NewFromConfig(cfg, s3compat.Options()), cfg.Region, nil |
| 114 | } |
| 115 | |
| 116 | // newReplicaClient builds litestream's replica client for the LTX files at |
| 117 | // s3://bucket/prefix. litestream reads its OWN env var (LITESTREAM_S3_ENDPOINT) |
| 118 | // only on the URL-parsing path, which this package never takes — it sets the |
| 119 | // fields directly — so the override is applied here too. Endpoint/ForcePathStyle |
| 120 | // is all it takes: litestream's Init derives the rest (the checksum knobs and |
| 121 | // the non-AWS default region) from a non-empty Endpoint on its own, and the |
| 122 | // signed-payload and content-MD5 defaults NewReplicaClient already ships are |
| 123 | // the ones every compatible in litestream's provider table wants. |
| 124 | func newReplicaClient(bucket, prefix, region string) (*lss3.ReplicaClient, error) { |
| 125 | return newReplicaClientForUID(bucket, prefix, region, os.Geteuid(), rootaws.OffcloudCredentials) |
| 126 | } |
| 127 | |
| 128 | var errRootReplicaSessionToken = errors.New("root Litestream cannot use a session token from /etc/carlos/credentials. set the keys in the environment instead, or use an EC2 instance profile") |
| 129 | |
| 130 | func newReplicaClientForUID(bucket, prefix, region string, euid int, offcloud func() (aws.Credentials, error)) (*lss3.ReplicaClient, error) { |
| 131 | c := lss3.NewReplicaClient() |
| 132 | if euid == 0 { |
| 133 | creds, err := offcloud() |
| 134 | if err != nil { |
| 135 | return nil, err |
| 136 | } |
| 137 | if creds.SessionToken != "" { |
| 138 | return nil, errRootReplicaSessionToken |
| 139 | } |
| 140 | c.AccessKeyID = creds.AccessKeyID |
| 141 | c.SecretAccessKey = creds.SecretAccessKey |
| 142 | } |
| 143 | c.Bucket = bucket |
| 144 | c.Path = prefix |
| 145 | c.Region = region |
| 146 | if ep := s3Endpoint(); ep != "" { |
| 147 | c.Endpoint = ep |
| 148 | c.ForcePathStyle = true |
| 149 | } |
| 150 | return c, nil |
| 151 | } |
| 152 | |
| 153 | // ReadReplicaClient is newReplicaClient for READERS outside this package — |
| 154 | // the console's db door (spec 2026-09-02 §6). Identical construction on |
| 155 | // purpose, region resolution included (Restore's resolveRegion): a second |
| 156 | // hand-rolled client is how the endpoint override, the path-style flag and |
| 157 | // the signing region drift apart between the writer and the reader of the |
| 158 | // same bucket. |
| 159 | // |
| 160 | // It returns the same type the writer uses, so nothing here can enforce |
| 161 | // read-only by construction; that guarantee lives in internal/dbdoor's |
| 162 | // client, which has no write methods at all, and in the door only ever |
| 163 | // calling LTXFiles and OpenLTXFile on what this returns. |
| 164 | func ReadReplicaClient(ctx context.Context, bucket, prefix, region string) (*lss3.ReplicaClient, error) { |
| 165 | region, err := resolveRegion(ctx, region) |
| 166 | if err != nil { |
| 167 | return nil, err |
| 168 | } |
| 169 | return newReplicaClient(bucket, prefix, region) |
| 170 | } |
| 171 | |
amadan