| 1 | package s3compat |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "net/http" |
| 6 | "net/http/httptest" |
| 7 | "strings" |
| 8 | "sync" |
| 9 | "testing" |
| 10 | |
| 11 | "github.com/aws/aws-sdk-go-v2/aws" |
| 12 | "github.com/aws/aws-sdk-go-v2/credentials" |
| 13 | awss3 "github.com/aws/aws-sdk-go-v2/service/s3" |
| 14 | ) |
| 15 | |
| 16 | // recorder is a fake S3 that records the conditional headers it was sent. |
| 17 | type recorder struct { |
| 18 | mu sync.Mutex |
| 19 | seen map[string]string // "METHOD If-Match" -> value |
| 20 | auth map[string]string // "METHOD" -> Authorization |
| 21 | } |
| 22 | |
| 23 | func newRecorder(t *testing.T) (*recorder, string) { |
| 24 | t.Helper() |
| 25 | r := &recorder{seen: map[string]string{}, auth: map[string]string{}} |
| 26 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { |
| 27 | r.mu.Lock() |
| 28 | r.seen[req.Method+" If-Match"] = req.Header.Get("If-Match") |
| 29 | r.auth[req.Method] = req.Header.Get("Authorization") |
| 30 | r.mu.Unlock() |
| 31 | w.Header().Set("ETag", `"deadbeef"`) |
| 32 | w.WriteHeader(http.StatusOK) |
| 33 | })) |
| 34 | t.Cleanup(srv.Close) |
| 35 | return r, srv.URL |
| 36 | } |
| 37 | |
| 38 | func (r *recorder) get(k string) string { |
| 39 | r.mu.Lock() |
| 40 | defer r.mu.Unlock() |
| 41 | return r.seen[k] |
| 42 | } |
| 43 | |
| 44 | func (r *recorder) authFor(m string) string { |
| 45 | r.mu.Lock() |
| 46 | defer r.mu.Unlock() |
| 47 | return r.auth[m] |
| 48 | } |
| 49 | |
| 50 | func testClient(t *testing.T) *awss3.Client { |
| 51 | t.Helper() |
| 52 | cfg := aws.Config{ |
| 53 | Region: "nbg1", |
| 54 | Credentials: credentials.NewStaticCredentialsProvider("k", "s", ""), |
| 55 | } |
| 56 | return awss3.NewFromConfig(cfg, Options()) |
| 57 | } |
| 58 | |
| 59 | // TestUnquotedIfMatch_RewritesPutOnly is the core of the fix: a PutObject's |
| 60 | // If-Match must reach the wire unquoted (what Ceph RGW compares against), |
| 61 | // while a DeleteObject's must be left exactly as the SDK wrote it — the quoted |
| 62 | // form is what works for conditional DELETE on the same store, and a working |
| 63 | // path is not worth touching. |
| 64 | func TestUnquotedIfMatch_RewritesPutOnly(t *testing.T) { |
| 65 | rec, url := newRecorder(t) |
| 66 | t.Setenv("CARLOS_S3_ENDPOINT", url) |
| 67 | // The httptest host is 127.0.0.1, not a Hetzner hostname, so force the |
| 68 | // quirk on by name — which also exercises the CARLOS_S3_QUIRKS escape |
| 69 | // hatch that lets an operator try a fix against an unrecognised store. |
| 70 | t.Setenv("CARLOS_S3_QUIRKS", "unquoted-if-match") |
| 71 | |
| 72 | c := testClient(t) |
| 73 | ctx := context.Background() |
| 74 | quoted := `"deadbeef"` |
| 75 | |
| 76 | if _, err := c.PutObject(ctx, &awss3.PutObjectInput{ |
| 77 | Bucket: aws.String("b"), Key: aws.String("k"), |
| 78 | Body: strings.NewReader("x"), IfMatch: aws.String(quoted), |
| 79 | }); err != nil { |
| 80 | t.Fatalf("PutObject: %v", err) |
| 81 | } |
| 82 | if got := rec.get("PUT If-Match"); got != "deadbeef" { |
| 83 | t.Errorf("PUT If-Match = %q, want %q (quotes stripped)", got, "deadbeef") |
| 84 | } |
| 85 | |
| 86 | if _, err := c.DeleteObject(ctx, &awss3.DeleteObjectInput{ |
| 87 | Bucket: aws.String("b"), Key: aws.String("k"), IfMatch: aws.String(quoted), |
| 88 | }); err != nil { |
| 89 | t.Fatalf("DeleteObject: %v", err) |
| 90 | } |
| 91 | if got := rec.get("DELETE If-Match"); got != quoted { |
| 92 | t.Errorf("DELETE If-Match = %q, want %q (left untouched)", got, quoted) |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | // TestUnquotedIfMatch_RewriteIsSigned guards the subtlety that makes this a |
| 97 | // Build-step middleware rather than a Finalize one: If-Match is a SIGNED |
| 98 | // header, so a rewrite after signing would produce SignatureDoesNotMatch |
| 99 | // instead of a fix. If the signer did not see if-match, it is not in |
| 100 | // SignedHeaders — which is observable from the Authorization header. |
| 101 | func TestUnquotedIfMatch_RewriteIsSigned(t *testing.T) { |
| 102 | rec, url := newRecorder(t) |
| 103 | t.Setenv("CARLOS_S3_ENDPOINT", url) |
| 104 | t.Setenv("CARLOS_S3_QUIRKS", "unquoted-if-match") |
| 105 | |
| 106 | c := testClient(t) |
| 107 | if _, err := c.PutObject(context.Background(), &awss3.PutObjectInput{ |
| 108 | Bucket: aws.String("b"), Key: aws.String("k"), |
| 109 | Body: strings.NewReader("x"), IfMatch: aws.String(`"deadbeef"`), |
| 110 | }); err != nil { |
| 111 | t.Fatalf("PutObject: %v", err) |
| 112 | } |
| 113 | auth := rec.authFor("PUT") |
| 114 | if !strings.Contains(auth, "SignedHeaders=") { |
| 115 | t.Fatalf("no SignedHeaders in Authorization: %q", auth) |
| 116 | } |
| 117 | if !strings.Contains(auth, "if-match") { |
| 118 | t.Errorf("if-match missing from SignedHeaders — the rewrite is outside the signature: %q", auth) |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | // TestOptions_NoOpWithoutEndpoint is the flagship guard: on stock AWS this |
| 123 | // package must change nothing at all, including registering no middleware. |
| 124 | func TestOptions_NoOpWithoutEndpoint(t *testing.T) { |
| 125 | t.Setenv("CARLOS_S3_ENDPOINT", "") |
| 126 | t.Setenv("CARLOS_S3_QUIRKS", "") |
| 127 | |
| 128 | var o awss3.Options |
| 129 | Options()(&o) |
| 130 | |
| 131 | if o.BaseEndpoint != nil { |
| 132 | t.Errorf("BaseEndpoint = %q, want nil on stock AWS", *o.BaseEndpoint) |
| 133 | } |
| 134 | if o.UsePathStyle { |
| 135 | t.Error("UsePathStyle = true, want false on stock AWS") |
| 136 | } |
| 137 | if len(o.APIOptions) != 0 { |
| 138 | t.Errorf("APIOptions = %d, want 0 — no quirk middleware on stock AWS", len(o.APIOptions)) |
| 139 | } |
| 140 | if o.RequestChecksumCalculation != 0 { |
| 141 | t.Errorf("RequestChecksumCalculation = %v, want the SDK default on stock AWS", o.RequestChecksumCalculation) |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | // TestOptions_UniversalSettingsUnderEndpoint pins what every compatible store |
| 146 | // gets regardless of provider. |
| 147 | func TestOptions_UniversalSettingsUnderEndpoint(t *testing.T) { |
| 148 | t.Setenv("CARLOS_S3_ENDPOINT", "https://nbg1.your-objectstorage.com") |
| 149 | t.Setenv("CARLOS_S3_QUIRKS", "") |
| 150 | |
| 151 | var o awss3.Options |
| 152 | Options()(&o) |
| 153 | |
| 154 | if o.BaseEndpoint == nil || *o.BaseEndpoint != "https://nbg1.your-objectstorage.com" { |
| 155 | t.Errorf("BaseEndpoint = %v, want the override", o.BaseEndpoint) |
| 156 | } |
| 157 | if !o.UsePathStyle { |
| 158 | t.Error("UsePathStyle = false, want true under an override") |
| 159 | } |
| 160 | if o.RequestChecksumCalculation != aws.RequestChecksumCalculationWhenRequired { |
| 161 | t.Error("RequestChecksumCalculation should be when-required under an override") |
| 162 | } |
| 163 | if len(o.APIOptions) != 1 { |
| 164 | t.Errorf("APIOptions = %d, want 1 (the Hetzner If-Match quirk matched by host)", len(o.APIOptions)) |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | func TestEndpointHost(t *testing.T) { |
| 169 | for _, tt := range []struct{ ep, want string }{ |
| 170 | {"https://nbg1.your-objectstorage.com", "nbg1.your-objectstorage.com"}, |
| 171 | {"nbg1.your-objectstorage.com", "nbg1.your-objectstorage.com"}, // no scheme |
| 172 | {"http://localhost:9000", "localhost"}, |
| 173 | {"", ""}, |
| 174 | } { |
| 175 | t.Setenv("CARLOS_S3_ENDPOINT", tt.ep) |
| 176 | if got := EndpointHost(); got != tt.want { |
| 177 | t.Errorf("EndpointHost() with %q = %q, want %q", tt.ep, got, tt.want) |
| 178 | } |
| 179 | } |
| 180 | } |
| 181 | |
| 182 | // TestQuirkMatching keeps the table honest: a quirk measured on one provider |
| 183 | // must not leak onto stores nobody has tested it against. |
| 184 | func TestQuirkMatching(t *testing.T) { |
| 185 | t.Setenv("CARLOS_S3_QUIRKS", "") |
| 186 | for _, tt := range []struct { |
| 187 | host string |
| 188 | want bool |
| 189 | }{ |
| 190 | {"nbg1.your-objectstorage.com", true}, |
| 191 | {"fsn1.your-objectstorage.com", true}, |
| 192 | {"s3.eu-west-1.amazonaws.com", false}, |
| 193 | {"minio.internal", false}, |
| 194 | {"evil-your-objectstorage.com", false}, // suffix match needs the dot |
| 195 | } { |
| 196 | if got := unquotedIfMatch.enabled(tt.host); got != tt.want { |
| 197 | t.Errorf("unquoted-if-match enabled for %q = %v, want %v", tt.host, got, tt.want) |
| 198 | } |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | // TestQuirkForceEnable covers the escape hatch: trying a known fix against an |
| 203 | // unrecognised store should not need a rebuild. |
| 204 | func TestQuirkForceEnable(t *testing.T) { |
| 205 | t.Setenv("CARLOS_S3_QUIRKS", "unquoted-if-match") |
| 206 | if !unquotedIfMatch.enabled("some.unknown.store") { |
| 207 | t.Error("CARLOS_S3_QUIRKS did not force the quirk on for an unmatched host") |
| 208 | } |
| 209 | t.Setenv("CARLOS_S3_QUIRKS", " other-quirk , unquoted-if-match ") |
| 210 | if !unquotedIfMatch.enabled("some.unknown.store") { |
| 211 | t.Error("CARLOS_S3_QUIRKS should tolerate spaces and a list") |
| 212 | } |
| 213 | t.Setenv("CARLOS_S3_QUIRKS", "something-else") |
| 214 | if unquotedIfMatch.enabled("some.unknown.store") { |
| 215 | t.Error("an unrelated CARLOS_S3_QUIRKS entry must not enable this quirk") |
| 216 | } |
| 217 | } |
| 218 | |
| 219 | // TestQuirkNamesAreUnique — the table is looked up by name from an env var, so |
| 220 | // duplicates would make CARLOS_S3_QUIRKS ambiguous. |
| 221 | func TestQuirkNamesAreUnique(t *testing.T) { |
| 222 | seen := map[string]bool{} |
| 223 | for _, q := range Quirks { |
| 224 | if q.Name == "" { |
| 225 | t.Error("a quirk has no Name; CARLOS_S3_QUIRKS could not address it") |
| 226 | } |
| 227 | if q.Reason == "" { |
| 228 | t.Errorf("quirk %q has no Reason — the table exists to record WHY", q.Name) |
| 229 | } |
| 230 | if seen[q.Name] { |
| 231 | t.Errorf("duplicate quirk name %q", q.Name) |
| 232 | } |
| 233 | seen[q.Name] = true |
| 234 | } |
| 235 | } |
| 236 | |
amadan