| 1 | // Package s3compat is the one place CARLOS records how a given S3-compatible |
| 2 | // store differs from AWS S3, and the only place that knows CARLOS_S3_ENDPOINT. |
| 3 | // |
| 4 | // Two kinds of adjustment live here. The first is universal — anything that is |
| 5 | // not AWS needs a base endpoint, path-style addressing and when-required |
| 6 | // checksums (default-on flexible checksums wrap bodies in aws-chunked framing |
| 7 | // some compatibles reject). Those were already duplicated in |
| 8 | // release.NewS3Store and internal/lease; Options folds them into one. |
| 9 | // |
| 10 | // The second is per-provider QUIRKS: places a store answers differently enough |
| 11 | // that correct S3 code breaks against it. Those go in the quirks table below, |
| 12 | // each entry naming the provider it matches, what it changes, and — the part |
| 13 | // that matters six months later — the measurement that justifies it. Adding a |
| 14 | // provider is one entry; nothing else in the codebase changes. |
| 15 | // |
| 16 | // Deliberately NOT covered: litestream's replica client, which builds its own |
| 17 | // S3 client we cannot decorate. It needs none of the quirks here — it writes |
| 18 | // LTX files with unconditional PUTs — and its own provider auto-detection |
| 19 | // already handles the universal settings when an Endpoint is set. The leaser |
| 20 | // is different: it takes a ready-made client (lss3.Leaser.SetClient), which is |
| 21 | // exactly how the If-Match quirk below reaches it. |
| 22 | package s3compat |
| 23 | |
| 24 | import ( |
| 25 | "context" |
| 26 | "net/url" |
| 27 | "os" |
| 28 | "strings" |
| 29 | |
| 30 | "github.com/aws/aws-sdk-go-v2/aws" |
| 31 | awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" |
| 32 | awss3 "github.com/aws/aws-sdk-go-v2/service/s3" |
| 33 | "github.com/aws/smithy-go/middleware" |
| 34 | smithyhttp "github.com/aws/smithy-go/transport/http" |
| 35 | ) |
| 36 | |
| 37 | // Endpoint reports the CARLOS_S3_ENDPOINT override, or "" for stock AWS. Read |
| 38 | // per call rather than cached at init so a test — and a process that re-reads |
| 39 | // its config — sees the value it just set. |
| 40 | func Endpoint() string { |
| 41 | return strings.TrimSpace(os.Getenv("CARLOS_S3_ENDPOINT")) |
| 42 | } |
| 43 | |
| 44 | // EndpointHost is the bare host of the override ("" for stock AWS), which is |
| 45 | // what quirk matchers key on. A bare host with no scheme still parses: the |
| 46 | // value is normalised to a URL first, so both "https://x.example" and |
| 47 | // "x.example" match the same way. |
| 48 | func EndpointHost() string { |
| 49 | ep := Endpoint() |
| 50 | if ep == "" { |
| 51 | return "" |
| 52 | } |
| 53 | if !strings.Contains(ep, "://") { |
| 54 | ep = "https://" + ep |
| 55 | } |
| 56 | u, err := url.Parse(ep) |
| 57 | if err != nil { |
| 58 | return "" |
| 59 | } |
| 60 | return u.Hostname() |
| 61 | } |
| 62 | |
| 63 | // Options returns the S3 client option every CARLOS S3 client should apply. |
| 64 | // With no override it is a NO-OP — the flagship's pure-AWS path must stay |
| 65 | // byte-identical, so nothing here may touch a client that talks to real S3. |
| 66 | func Options() func(*awss3.Options) { |
| 67 | return func(o *awss3.Options) { |
| 68 | ep := Endpoint() |
| 69 | if ep == "" { |
| 70 | return |
| 71 | } |
| 72 | o.BaseEndpoint = &ep |
| 73 | o.UsePathStyle = true |
| 74 | o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired |
| 75 | o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired |
| 76 | for _, q := range Quirks { |
| 77 | if q.enabled(EndpointHost()) { |
| 78 | q.Apply(o) |
| 79 | } |
| 80 | } |
| 81 | } |
| 82 | } |
| 83 | |
| 84 | // Quirk is one provider's documented deviation from S3 and the fix for it. |
| 85 | type Quirk struct { |
| 86 | // Name is the stable identifier, also the token CARLOS_S3_QUIRKS uses to |
| 87 | // force the quirk on for a store whose hostname Match does not know. |
| 88 | Name string |
| 89 | // Reason is why this exists — ideally the measurement, not a guess. |
| 90 | Reason string |
| 91 | // Match reports whether the quirk applies to an endpoint host. A nil Match |
| 92 | // means "every non-AWS store", which should be rare: prefer naming the |
| 93 | // providers actually measured, so an unmeasured store fails loudly in its |
| 94 | // own way rather than silently inheriting someone else's workaround. |
| 95 | Match func(host string) bool |
| 96 | // Apply adjusts the client. Wire-level fixes register SDK middleware here. |
| 97 | Apply func(*awss3.Options) |
| 98 | } |
| 99 | |
| 100 | func (q Quirk) enabled(host string) bool { |
| 101 | for _, forced := range strings.Split(os.Getenv("CARLOS_S3_QUIRKS"), ",") { |
| 102 | if strings.TrimSpace(forced) == q.Name { |
| 103 | return true |
| 104 | } |
| 105 | } |
| 106 | if q.Match == nil { |
| 107 | return true |
| 108 | } |
| 109 | return q.Match(host) |
| 110 | } |
| 111 | |
| 112 | // Quirks is the table. Add a provider by adding an entry. |
| 113 | var Quirks = []Quirk{unquotedIfMatch} |
| 114 | |
| 115 | // isHetzner matches Hetzner Object Storage (Ceph RGW), e.g. |
| 116 | // nbg1.your-objectstorage.com. Same suffix litestream's own |
| 117 | // IsHetznerEndpoint uses. |
| 118 | func isHetzner(host string) bool { |
| 119 | return strings.HasSuffix(host, ".your-objectstorage.com") |
| 120 | } |
| 121 | |
| 122 | // unquotedIfMatch is the fix for the compare-and-swap failure measured against |
| 123 | // Hetzner Object Storage on 2026-08-07. |
| 124 | // |
| 125 | // RGW compares If-Match against its UNQUOTED internal etag, so the quoted |
| 126 | // entity-tag form that S3 returns — and that the AWS SDK faithfully round-trips |
| 127 | // — never matches. Measured against the carlos bucket: |
| 128 | // |
| 129 | // PUT If-Match: "<correct etag>" -> 412 (quoted, as AWS returns it) |
| 130 | // PUT If-Match: <correct etag> -> OK (same etag, unquoted) |
| 131 | // PUT If-Match: <wrong etag> -> 412 (still correctly rejected) |
| 132 | // PUT If-None-Match: * (exists) -> 412 (correctly rejected) |
| 133 | // |
| 134 | // The third line is what makes this safe: the condition IS evaluated, only the |
| 135 | // quoting form differs, so stripping the quotes fixes compare-and-swap rather |
| 136 | // than disabling it. RFC 9110 entity-tags are quoted, so AWS is right and RGW |
| 137 | // is wrong — but CARLOS still has to interoperate. |
| 138 | // |
| 139 | // Scoped to PutObject on purpose. Conditional DELETE accepts the quoted form |
| 140 | // on the same store (it is how the lease releases today, and it works), and a |
| 141 | // working path is not worth touching on the strength of an untested guess. |
| 142 | // |
| 143 | // What it unblocks: every CAS replace in CARLOS — release pointers and |
| 144 | // manifests, the ledger head, appconfig generations, provision records, status |
| 145 | // snapshots, metrics rollups — plus litestream's lease heartbeat, which renews |
| 146 | // with PUT If-Match every TTL/3 and so lost the lease at the first renewal, |
| 147 | // standing a woken instance back down five minutes later, every time. |
| 148 | var unquotedIfMatch = Quirk{ |
| 149 | Name: "unquoted-if-match", |
| 150 | Reason: "Ceph RGW (Hetzner) compares If-Match against the unquoted etag; the quoted form always 412s", |
| 151 | Match: isHetzner, |
| 152 | Apply: func(o *awss3.Options) { |
| 153 | o.APIOptions = append(o.APIOptions, unquoteIfMatchMiddleware) |
| 154 | }, |
| 155 | } |
| 156 | |
| 157 | // unquoteIfMatchMiddleware strips the surrounding quotes from a PutObject's |
| 158 | // If-Match header. |
| 159 | // |
| 160 | // It runs in the BUILD step, which is after serialization has written the |
| 161 | // header and before the Finalize step signs the request — the only window |
| 162 | // where the change is both possible and covered by the signature. Rewriting |
| 163 | // after signing would produce a SignatureDoesNotMatch instead of a fix. |
| 164 | func unquoteIfMatchMiddleware(stack *middleware.Stack) error { |
| 165 | return stack.Build.Add(middleware.BuildMiddlewareFunc("carlosUnquoteIfMatch", |
| 166 | func(ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler) ( |
| 167 | middleware.BuildOutput, middleware.Metadata, error, |
| 168 | ) { |
| 169 | if awsmiddleware.GetOperationName(ctx) == "PutObject" { |
| 170 | if req, ok := in.Request.(*smithyhttp.Request); ok { |
| 171 | if v := req.Header.Get("If-Match"); v != "" { |
| 172 | if u := strings.Trim(v, `"`); u != v { |
| 173 | req.Header.Set("If-Match", u) |
| 174 | } |
| 175 | } |
| 176 | } |
| 177 | } |
| 178 | return next.HandleBuild(ctx, in) |
| 179 | }, |
| 180 | ), middleware.After) |
| 181 | } |
| 182 | |
amadan