| 1 | package edge |
| 2 | |
| 3 | // Task 7 of the multi-region ingress plan: the acceptance test the whole of |
| 4 | // certsync.go exists for — ONE bucket, TWO edges, one certificate. |
| 5 | // |
| 6 | // Edge A holds the housekeeper gate and is the only box allowed to talk to a |
| 7 | // CA. Edge B holds nothing, may never order, and yet has to (a) answer the |
| 8 | // HTTP-01 challenge for an order A is running and (b) serve the certificate A |
| 9 | // minted. Everything between the two is the control bucket's control/certs/ |
| 10 | // prefix and nothing else: no RPC, no gossip, no second writer. |
| 11 | // |
| 12 | // The CA here is a fake one (fakeACME below) rather than a mock of the ACME |
| 13 | // client, because the property under test is not "autocert was called" but |
| 14 | // "an independent validator, reaching a DIFFERENT box than the one that |
| 15 | // ordered, got the right key authorization and issued a certificate that a |
| 16 | // real TLS client will accept". A mock cannot fail that; a validator can. |
| 17 | // It is deliberately pointed at edge B's :80 handler and NEVER at edge A's — |
| 18 | // A's handler is served too, purely so the test can assert it received |
| 19 | // nothing. |
| 20 | // |
| 21 | // Conventions borrowed from the neighbouring harnesses (tunnel_e2e_test.go, |
| 22 | // fleet_e2e_test.go): real loopback listeners, in-process components wired |
| 23 | // the way edge.go's RunWithActivatorReady wires them, and bounds asserted |
| 24 | // rather than narrated. Helpers already in this package (testIssuerEdge, |
| 25 | // ecdsaHello, offlineManager, hostCertPEM, putBucket, localCert) are reused |
| 26 | // rather than re-cut. |
| 27 | |
| 28 | import ( |
| 29 | "bytes" |
| 30 | "context" |
| 31 | "crypto/ecdsa" |
| 32 | "crypto/elliptic" |
| 33 | "crypto/rand" |
| 34 | "crypto/tls" |
| 35 | "crypto/x509" |
| 36 | "crypto/x509/pkix" |
| 37 | "encoding/base64" |
| 38 | "encoding/json" |
| 39 | "encoding/pem" |
| 40 | "errors" |
| 41 | "fmt" |
| 42 | "io" |
| 43 | "log" |
| 44 | "math/big" |
| 45 | "net" |
| 46 | "net/http" |
| 47 | "net/http/httptest" |
| 48 | "os" |
| 49 | "slices" |
| 50 | "strings" |
| 51 | "sync" |
| 52 | "sync/atomic" |
| 53 | "testing" |
| 54 | "time" |
| 55 | |
| 56 | "golang.org/x/crypto/acme" |
| 57 | "golang.org/x/crypto/acme/autocert" |
| 58 | |
| 59 | "github.com/carlosframework/platform/internal/release" |
| 60 | ) |
| 61 | |
| 62 | const ( |
| 63 | // e2eHost is the name edge A mints for and edge B serves. A .test TLD, so |
| 64 | // nothing here can resolve or be reached for real. |
| 65 | e2eHost = "app.bab.example.test" |
| 66 | // e2eUnmintedHost is in both edges' registries but is never issued: it is |
| 67 | // what the gated twin is asked to mint, so the refusal is the GATE's and |
| 68 | // not "that host isn't mine". |
| 69 | e2eUnmintedHost = "other.bab.example.test" |
| 70 | // e2eLocalOnlyHost exists only in edge B's own on-disk cache — yesterday's |
| 71 | // material, from before the election moved. Nothing publishes it. |
| 72 | e2eLocalOnlyHost = "old.bab.example.test" |
| 73 | ) |
| 74 | |
| 75 | // ---- the fake ACME CA -------------------------------------------------------- |
| 76 | |
| 77 | // fakeACME is a minimal RFC 8555 CA: directory, nonce, account, order, one |
| 78 | // http-01 authorization, finalize, certificate. It is NOT a security model — |
| 79 | // it never verifies a JWS signature and never checks a nonce, because neither |
| 80 | // is under test. What it DOES check is the only thing that matters here: that |
| 81 | // the HTTP-01 response body served back to it equals token + "." + |
| 82 | // JWK-thumbprint of the account key that placed the order. That check is what |
| 83 | // makes "edge B answered edge A's challenge" a proof rather than a narration. |
| 84 | // |
| 85 | // It offers ONLY an http-01 challenge. autocert prefers tls-alpn-01 and falls |
| 86 | // back to http-01 after a whole failed order (autocert.go's |
| 87 | // AuthorizeOrderLoop); offering one type keeps the run to a single order, so |
| 88 | // the order counter below is an exact count of "who talked to a CA". |
| 89 | type fakeACME struct { |
| 90 | srv *httptest.Server |
| 91 | rootKey *ecdsa.PrivateKey |
| 92 | root *x509.Certificate |
| 93 | |
| 94 | mu sync.Mutex |
| 95 | // validateVia is the base URL the CA fetches HTTP-01 challenges from — |
| 96 | // set to edge B's :80 server, never edge A's. |
| 97 | validateVia string |
| 98 | // thumb is the account key's JWK thumbprint, learned from the JWS |
| 99 | // protected header of the new-account request. |
| 100 | thumb string |
| 101 | // accounts and orders are the counters the test asserts on: "B started |
| 102 | // zero ACME orders" is this number not moving. |
| 103 | accounts int |
| 104 | orders int |
| 105 | seq int |
| 106 | byID map[string]*fakeACMEOrder |
| 107 | // validations records every challenge fetch the CA made, in order. |
| 108 | validations []fakeACMEValidation |
| 109 | // faults collects anything the CA found wrong. Never t.Fatalf: these |
| 110 | // handlers run on the server's goroutines and, for deactivatePendingAuthz, |
| 111 | // can outlive the test. |
| 112 | faults []string |
| 113 | } |
| 114 | |
| 115 | type fakeACMEOrder struct { |
| 116 | domain string |
| 117 | token string |
| 118 | authz string // pending | valid | invalid |
| 119 | status string // pending | ready | valid |
| 120 | chain []byte // PEM, once finalized |
| 121 | } |
| 122 | |
| 123 | // fakeACMEValidation is one HTTP-01 fetch: where it went, what Host it |
| 124 | // carried, the order's token, and whether the body was the right key |
| 125 | // authorization. |
| 126 | type fakeACMEValidation struct { |
| 127 | url string |
| 128 | host string |
| 129 | token string |
| 130 | ok bool |
| 131 | } |
| 132 | |
| 133 | func newFakeACME(t *testing.T) *fakeACME { |
| 134 | t.Helper() |
| 135 | key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) |
| 136 | if err != nil { |
| 137 | t.Fatal(err) |
| 138 | } |
| 139 | tmpl := &x509.Certificate{ |
| 140 | SerialNumber: big.NewInt(1), |
| 141 | Subject: pkix.Name{CommonName: "CARLOS test CA"}, |
| 142 | NotBefore: time.Now().Add(-time.Hour), |
| 143 | NotAfter: time.Now().Add(365 * 24 * time.Hour), |
| 144 | KeyUsage: x509.KeyUsageCertSign, |
| 145 | BasicConstraintsValid: true, |
| 146 | IsCA: true, |
| 147 | } |
| 148 | der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) |
| 149 | if err != nil { |
| 150 | t.Fatal(err) |
| 151 | } |
| 152 | root, err := x509.ParseCertificate(der) |
| 153 | if err != nil { |
| 154 | t.Fatal(err) |
| 155 | } |
| 156 | ca := &fakeACME{rootKey: key, root: root, byID: map[string]*fakeACMEOrder{}} |
| 157 | |
| 158 | mux := http.NewServeMux() |
| 159 | mux.HandleFunc("/directory", ca.serveDirectory) |
| 160 | mux.HandleFunc("/nonce", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) |
| 161 | mux.HandleFunc("/new-account", ca.serveNewAccount) |
| 162 | mux.HandleFunc("/new-order", ca.serveNewOrder) |
| 163 | mux.HandleFunc("/authz/", ca.serveAuthorization) |
| 164 | mux.HandleFunc("/chal/", ca.serveChallengeAccept) |
| 165 | mux.HandleFunc("/order/", ca.serveOrder) |
| 166 | mux.HandleFunc("/finalize/", ca.serveFinalize) |
| 167 | mux.HandleFunc("/cert/", ca.serveCert) |
| 168 | // Every ACME response carries a fresh Replay-Nonce; the client will not |
| 169 | // make a second POST without one. |
| 170 | ca.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 171 | w.Header().Set("Replay-Nonce", ca.nextNonce()) |
| 172 | mux.ServeHTTP(w, r) |
| 173 | })) |
| 174 | t.Cleanup(ca.srv.Close) |
| 175 | return ca |
| 176 | } |
| 177 | |
| 178 | func (ca *fakeACME) directoryURL() string { return ca.srv.URL + "/directory" } |
| 179 | |
| 180 | func (ca *fakeACME) url(path string) string { return ca.srv.URL + path } |
| 181 | |
| 182 | // validateAgainst points the CA's HTTP-01 validator at one edge's :80 server. |
| 183 | func (ca *fakeACME) validateAgainst(base string) { |
| 184 | ca.mu.Lock() |
| 185 | defer ca.mu.Unlock() |
| 186 | ca.validateVia = base |
| 187 | } |
| 188 | |
| 189 | func (ca *fakeACME) nextNonce() string { |
| 190 | ca.mu.Lock() |
| 191 | defer ca.mu.Unlock() |
| 192 | ca.seq++ |
| 193 | return fmt.Sprintf("nonce-%d", ca.seq) |
| 194 | } |
| 195 | |
| 196 | func (ca *fakeACME) counts() (accounts, orders int) { |
| 197 | ca.mu.Lock() |
| 198 | defer ca.mu.Unlock() |
| 199 | return ca.accounts, ca.orders |
| 200 | } |
| 201 | |
| 202 | func (ca *fakeACME) lastValidation() (fakeACMEValidation, bool) { |
| 203 | ca.mu.Lock() |
| 204 | defer ca.mu.Unlock() |
| 205 | if len(ca.validations) == 0 { |
| 206 | return fakeACMEValidation{}, false |
| 207 | } |
| 208 | return ca.validations[len(ca.validations)-1], true |
| 209 | } |
| 210 | |
| 211 | func (ca *fakeACME) recordFault(format string, args ...any) { |
| 212 | ca.mu.Lock() |
| 213 | defer ca.mu.Unlock() |
| 214 | ca.faults = append(ca.faults, fmt.Sprintf(format, args...)) |
| 215 | } |
| 216 | |
| 217 | // checkFaults fails the test with anything the CA rejected along the way — a |
| 218 | // malformed JWS, a CSR for the wrong name, an unknown order. Called at the end |
| 219 | // of each phase so a subtle protocol mistake surfaces as itself rather than as |
| 220 | // a timeout. |
| 221 | func (ca *fakeACME) checkFaults(t *testing.T) { |
| 222 | t.Helper() |
| 223 | ca.mu.Lock() |
| 224 | faults := append([]string(nil), ca.faults...) |
| 225 | ca.mu.Unlock() |
| 226 | for _, f := range faults { |
| 227 | t.Errorf("fake CA: %s", f) |
| 228 | } |
| 229 | } |
| 230 | |
| 231 | func (ca *fakeACME) rootPool() *x509.CertPool { |
| 232 | pool := x509.NewCertPool() |
| 233 | pool.AddCert(ca.root) |
| 234 | return pool |
| 235 | } |
| 236 | |
| 237 | func (ca *fakeACME) serveDirectory(w http.ResponseWriter, _ *http.Request) { |
| 238 | writeJSON(w, http.StatusOK, map[string]string{ |
| 239 | "newNonce": ca.url("/nonce"), |
| 240 | "newAccount": ca.url("/new-account"), |
| 241 | "newOrder": ca.url("/new-order"), |
| 242 | "revokeCert": ca.url("/revoke"), |
| 243 | "keyChange": ca.url("/key-change"), |
| 244 | }) |
| 245 | } |
| 246 | |
| 247 | func (ca *fakeACME) serveNewAccount(w http.ResponseWriter, r *http.Request) { |
| 248 | protected, _, err := decodeJWS(r) |
| 249 | if err != nil { |
| 250 | ca.recordFault("new-account: %v", err) |
| 251 | http.Error(w, "bad jws", http.StatusBadRequest) |
| 252 | return |
| 253 | } |
| 254 | raw, ok := protected["jwk"] |
| 255 | if !ok { |
| 256 | ca.recordFault("new-account: no jwk in the protected header (a KID-form registration?)") |
| 257 | http.Error(w, "no jwk", http.StatusBadRequest) |
| 258 | return |
| 259 | } |
| 260 | thumb, err := jwkThumbprintFromJSON(raw) |
| 261 | if err != nil { |
| 262 | ca.recordFault("new-account: %v", err) |
| 263 | http.Error(w, "bad jwk", http.StatusBadRequest) |
| 264 | return |
| 265 | } |
| 266 | ca.mu.Lock() |
| 267 | ca.accounts++ |
| 268 | n := ca.accounts |
| 269 | ca.thumb = thumb |
| 270 | ca.mu.Unlock() |
| 271 | w.Header().Set("Location", ca.url(fmt.Sprintf("/acct/%d", n))) |
| 272 | writeJSON(w, http.StatusCreated, map[string]any{"status": "valid"}) |
| 273 | } |
| 274 | |
| 275 | func (ca *fakeACME) serveNewOrder(w http.ResponseWriter, r *http.Request) { |
| 276 | _, payload, err := decodeJWS(r) |
| 277 | if err != nil { |
| 278 | ca.recordFault("new-order: %v", err) |
| 279 | http.Error(w, "bad jws", http.StatusBadRequest) |
| 280 | return |
| 281 | } |
| 282 | var req struct { |
| 283 | Identifiers []struct{ Type, Value string } `json:"identifiers"` |
| 284 | } |
| 285 | if err := json.Unmarshal(payload, &req); err != nil || len(req.Identifiers) != 1 { |
| 286 | ca.recordFault("new-order: want exactly one identifier, got %q (%v)", payload, err) |
| 287 | http.Error(w, "bad order", http.StatusBadRequest) |
| 288 | return |
| 289 | } |
| 290 | tokenBytes := make([]byte, 32) |
| 291 | if _, err := rand.Read(tokenBytes); err != nil { |
| 292 | ca.recordFault("new-order: %v", err) |
| 293 | http.Error(w, "no entropy", http.StatusInternalServerError) |
| 294 | return |
| 295 | } |
| 296 | ca.mu.Lock() |
| 297 | ca.orders++ |
| 298 | id := fmt.Sprintf("%d", ca.orders) |
| 299 | ca.byID[id] = &fakeACMEOrder{ |
| 300 | domain: req.Identifiers[0].Value, |
| 301 | // base64url of 32 bytes: 43 chars, which is what |
| 302 | // validChallengeToken's entropy floor is calibrated against. |
| 303 | token: base64.RawURLEncoding.EncodeToString(tokenBytes), |
| 304 | authz: "pending", |
| 305 | status: "pending", |
| 306 | } |
| 307 | ca.mu.Unlock() |
| 308 | w.Header().Set("Location", ca.url("/order/"+id)) |
| 309 | writeJSON(w, http.StatusCreated, ca.orderJSON(id)) |
| 310 | } |
| 311 | |
| 312 | // orderJSON renders an order in the shape acme.responseOrder parses. Caller |
| 313 | // must not hold ca.mu. |
| 314 | func (ca *fakeACME) orderJSON(id string) map[string]any { |
| 315 | ca.mu.Lock() |
| 316 | defer ca.mu.Unlock() |
| 317 | o := ca.byID[id] |
| 318 | if o == nil { |
| 319 | return map[string]any{"status": "invalid"} |
| 320 | } |
| 321 | body := map[string]any{ |
| 322 | "status": o.status, |
| 323 | "identifiers": []map[string]string{{"type": "dns", "value": o.domain}}, |
| 324 | "authorizations": []string{ca.url("/authz/" + id)}, |
| 325 | "finalize": ca.url("/finalize/" + id), |
| 326 | } |
| 327 | if o.status == "valid" { |
| 328 | body["certificate"] = ca.url("/cert/" + id) |
| 329 | } |
| 330 | return body |
| 331 | } |
| 332 | |
| 333 | func writeJSON(w http.ResponseWriter, code int, body any) { |
| 334 | w.Header().Set("Content-Type", "application/json") |
| 335 | w.WriteHeader(code) |
| 336 | _ = json.NewEncoder(w).Encode(body) |
| 337 | } |
| 338 | |
| 339 | // ---- JWS / JWK ----------------------------------------------------------- |
| 340 | |
| 341 | // decodeJWS pulls the protected header and the payload out of a flattened JWS. |
| 342 | // The signature is deliberately not verified — see fakeACME's doc. |
| 343 | func decodeJWS(r *http.Request) (map[string]json.RawMessage, []byte, error) { |
| 344 | raw, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) |
| 345 | if err != nil { |
| 346 | return nil, nil, fmt.Errorf("read body: %w", err) |
| 347 | } |
| 348 | var msg struct { |
| 349 | Protected string `json:"protected"` |
| 350 | Payload string `json:"payload"` |
| 351 | } |
| 352 | if err := json.Unmarshal(raw, &msg); err != nil { |
| 353 | return nil, nil, fmt.Errorf("not a JWS: %w", err) |
| 354 | } |
| 355 | phead, err := base64.RawURLEncoding.DecodeString(msg.Protected) |
| 356 | if err != nil { |
| 357 | return nil, nil, fmt.Errorf("protected header: %w", err) |
| 358 | } |
| 359 | var protected map[string]json.RawMessage |
| 360 | if err := json.Unmarshal(phead, &protected); err != nil { |
| 361 | return nil, nil, fmt.Errorf("protected header JSON: %w", err) |
| 362 | } |
| 363 | // POST-as-GET carries a zero-length payload. |
| 364 | payload, err := base64.RawURLEncoding.DecodeString(msg.Payload) |
| 365 | if err != nil { |
| 366 | return nil, nil, fmt.Errorf("payload: %w", err) |
| 367 | } |
| 368 | return protected, payload, nil |
| 369 | } |
| 370 | |
| 371 | // jwkThumbprintFromJSON reconstructs the P-256 public key from a JWK and hands |
| 372 | // it to acme.JWKThumbprint — the library's OWN canonicalization, so the |
| 373 | // expected key authorization is computed the way a real CA computes it rather |
| 374 | // than by hashing whatever bytes happened to be on the wire. |
| 375 | func jwkThumbprintFromJSON(raw json.RawMessage) (string, error) { |
| 376 | var jwk struct{ Crv, Kty, X, Y string } |
| 377 | if err := json.Unmarshal(raw, &jwk); err != nil { |
| 378 | return "", fmt.Errorf("jwk JSON: %w", err) |
| 379 | } |
| 380 | if jwk.Kty != "EC" || jwk.Crv != "P-256" { |
| 381 | return "", fmt.Errorf("jwk kty/crv = %q/%q, want EC/P-256 (autocert's account key)", jwk.Kty, jwk.Crv) |
| 382 | } |
| 383 | x, err := base64.RawURLEncoding.DecodeString(jwk.X) |
| 384 | if err != nil { |
| 385 | return "", fmt.Errorf("jwk x: %w", err) |
| 386 | } |
| 387 | y, err := base64.RawURLEncoding.DecodeString(jwk.Y) |
| 388 | if err != nil { |
| 389 | return "", fmt.Errorf("jwk y: %w", err) |
| 390 | } |
| 391 | return acme.JWKThumbprint(&ecdsa.PublicKey{ |
| 392 | Curve: elliptic.P256(), |
| 393 | X: new(big.Int).SetBytes(x), |
| 394 | Y: new(big.Int).SetBytes(y), |
| 395 | }) |
| 396 | } |
| 397 | |
| 398 | // ---- two edges ----------------------------------------------------------- |
| 399 | |
| 400 | // e2eEdge is one box, wired exactly as edge.go's RunWithActivatorReady wires |
| 401 | // it: the raw local cache kept aside, the publishing wrapper over it, the |
| 402 | // mirror built from the RAW handle, autocert's manager over the wrapper, and |
| 403 | // the two chains (handshake and :80) composed in the same order. |
| 404 | type e2eEdge struct { |
| 405 | edge *Edge |
| 406 | store *e2eRecordingStore |
| 407 | local autocert.Cache |
| 408 | mirror *certMirror |
| 409 | mgr *autocert.Manager |
| 410 | // getCert is the full handshake chain: mirror in front of autocert, with |
| 411 | // the direct local-cache last resort behind it. |
| 412 | getCert func(*tls.ClientHelloInfo) (*tls.Certificate, error) |
| 413 | // httpHandler is the full :80 ACME chain: the mirror's challenge handler |
| 414 | // in front of autocert's HTTPHandler. |
| 415 | httpHandler http.Handler |
| 416 | tlsConfig *tls.Config |
| 417 | // hits counts requests that reached this box's :80 chain. |
| 418 | hits atomic.Int64 |
| 419 | } |
| 420 | |
| 421 | // newE2EEdge builds one edge over its OWN handle on the shared bucket |
| 422 | // directory — two boxes, two clients, one bucket, which is what makes the |
| 423 | // disjointness assertion in the P6 subtest meaningful. |
| 424 | func newE2EEdge(t *testing.T, bucketDir string, holds bool, directoryURL string, hosts ...string) *e2eEdge { |
| 425 | t.Helper() |
| 426 | b := &e2eEdge{store: &e2eRecordingStore{Store: release.NewDirStore(bucketDir)}} |
| 427 | b.edge = testIssuerEdge(hosts...) |
| 428 | b.edge.staticStore = b.store |
| 429 | b.edge.issuer = func() (context.Context, bool) { |
| 430 | if !holds { |
| 431 | return nil, false |
| 432 | } |
| 433 | return context.Background(), true |
| 434 | } |
| 435 | |
| 436 | // The RAW cache, kept before the publishing wrapper goes on (edge.go). |
| 437 | b.local = autocert.DirCache(t.TempDir()) |
| 438 | cache := b.edge.armCertIssuance(b.local) |
| 439 | if _, wrapped := cache.(*publishingCache); !wrapped { |
| 440 | t.Fatal("a store-backed edge must publish every issuance artifact") |
| 441 | } |
| 442 | b.mirror = b.edge.newCertMirror(b.local) |
| 443 | if b.mirror == nil { |
| 444 | t.Fatal("a store-backed edge must have a mirror to serve from") |
| 445 | } |
| 446 | b.mgr = &autocert.Manager{ |
| 447 | Prompt: autocert.AcceptTOS, |
| 448 | Cache: cache, |
| 449 | HostPolicy: b.edge.issuerHostPolicy(), |
| 450 | Client: &acme.Client{DirectoryURL: directoryURL}, |
| 451 | } |
| 452 | b.tlsConfig = b.mgr.TLSConfig() |
| 453 | b.getCert = b.mirror.wrapGetCertificate(b.tlsConfig.GetCertificate) |
| 454 | b.tlsConfig.GetCertificate = b.getCert |
| 455 | |
| 456 | // HTTPHandler is what sets autocert's tryHTTP01, so it has to be built |
| 457 | // before any order starts — exactly as edge.go builds it before binding. |
| 458 | fallback := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { |
| 459 | w.WriteHeader(http.StatusTeapot) |
| 460 | }) |
| 461 | chain := b.mirror.challengeHandler(b.mgr.HTTPHandler(fallback)) |
| 462 | b.httpHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 463 | b.hits.Add(1) |
| 464 | chain.ServeHTTP(w, r) |
| 465 | }) |
| 466 | return b |
| 467 | } |
| 468 | |
| 469 | // serveHTTP puts this edge's :80 chain on a loopback listener and returns its |
| 470 | // base URL — what the CA is (or, for edge A, deliberately is not) pointed at. |
| 471 | func (b *e2eEdge) serveHTTP(t *testing.T) string { |
| 472 | t.Helper() |
| 473 | srv := httptest.NewServer(b.httpHandler) |
| 474 | t.Cleanup(srv.Close) |
| 475 | return srv.URL |
| 476 | } |
| 477 | |
| 478 | // serveTLS puts this edge's handshake chain on a loopback listener. Raw |
| 479 | // tls.Server rather than an HTTP server: the assertion is about the |
| 480 | // certificate the handshake produces, and nothing above it. |
| 481 | func (b *e2eEdge) serveTLS(t *testing.T) net.Listener { |
| 482 | t.Helper() |
| 483 | ln, err := net.Listen("tcp", "127.0.0.1:0") |
| 484 | if err != nil { |
| 485 | t.Fatal(err) |
| 486 | } |
| 487 | t.Cleanup(func() { _ = ln.Close() }) |
| 488 | cfg := b.tlsConfig |
| 489 | go func() { |
| 490 | for { |
| 491 | conn, err := ln.Accept() |
| 492 | if err != nil { |
| 493 | return |
| 494 | } |
| 495 | go func() { |
| 496 | tc := tls.Server(conn, cfg) |
| 497 | _ = tc.HandshakeContext(context.Background()) |
| 498 | _ = tc.Close() |
| 499 | }() |
| 500 | } |
| 501 | }() |
| 502 | return ln |
| 503 | } |
| 504 | |
| 505 | // e2eDial completes one handshake against ln with SNI name, verifying the |
| 506 | // chain against the fake CA's root — a real client's verification, not a |
| 507 | // peek at the certificate the server chose. |
| 508 | func e2eDial(ln net.Listener, name string, roots *x509.CertPool) (*tls.ConnectionState, error) { |
| 509 | conn, err := tls.Dial("tcp", ln.Addr().String(), &tls.Config{ |
| 510 | ServerName: name, |
| 511 | RootCAs: roots, |
| 512 | MinVersion: tls.VersionTLS12, |
| 513 | }) |
| 514 | if err != nil { |
| 515 | return nil, err |
| 516 | } |
| 517 | defer conn.Close() |
| 518 | state := conn.ConnectionState() |
| 519 | return &state, nil |
| 520 | } |
| 521 | |
| 522 | // e2eRecordingStore is one box's handle on the bucket, remembering every key |
| 523 | // it read and every key it wrote. Reads bound what the mirror touches (P6); |
| 524 | // writes are how "a gated box publishes nothing" is asserted without trusting |
| 525 | // the gate's own bookkeeping. |
| 526 | type e2eRecordingStore struct { |
| 527 | release.Store |
| 528 | mu sync.Mutex |
| 529 | reads []string |
| 530 | wrote []string |
| 531 | } |
| 532 | |
| 533 | func (s *e2eRecordingStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 534 | s.mu.Lock() |
| 535 | s.reads = append(s.reads, key) |
| 536 | s.mu.Unlock() |
| 537 | return s.Store.Get(ctx, key) |
| 538 | } |
| 539 | |
| 540 | func (s *e2eRecordingStore) List(ctx context.Context, prefix string) ([]string, error) { |
| 541 | s.mu.Lock() |
| 542 | s.reads = append(s.reads, prefix) |
| 543 | s.mu.Unlock() |
| 544 | return s.Store.List(ctx, prefix) |
| 545 | } |
| 546 | |
| 547 | func (s *e2eRecordingStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| 548 | s.mu.Lock() |
| 549 | s.wrote = append(s.wrote, key) |
| 550 | s.mu.Unlock() |
| 551 | return s.Store.Put(ctx, key, body, ifMatch) |
| 552 | } |
| 553 | |
| 554 | func (s *e2eRecordingStore) Delete(ctx context.Context, key string) error { |
| 555 | s.mu.Lock() |
| 556 | s.wrote = append(s.wrote, key) |
| 557 | s.mu.Unlock() |
| 558 | return s.Store.Delete(ctx, key) |
| 559 | } |
| 560 | |
| 561 | func (s *e2eRecordingStore) snapshot() (reads, wrote []string) { |
| 562 | s.mu.Lock() |
| 563 | defer s.mu.Unlock() |
| 564 | return append([]string(nil), s.reads...), append([]string(nil), s.wrote...) |
| 565 | } |
| 566 | |
| 567 | // e2eQuietLogs sends the edge's own log lines to the void. The chain under |
| 568 | // test is deliberately chatty (publishing armed, mirror armed, serving from |
| 569 | // the local cache); the test's output is the assertions. |
| 570 | func e2eQuietLogs(t *testing.T) { |
| 571 | t.Helper() |
| 572 | log.SetOutput(io.Discard) |
| 573 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 574 | } |
| 575 | |
| 576 | // ---- P1/P2/P3/P6: the two-edge scenario -------------------------------------- |
| 577 | |
| 578 | func TestCertSyncE2EMintOnOneServeOnTheOther(t *testing.T) { |
| 579 | e2eQuietLogs(t) |
| 580 | bucket := t.TempDir() |
| 581 | ca := newFakeACME(t) |
| 582 | |
| 583 | // Edge B first: the CA has to be pointed at it before edge A places an |
| 584 | // order, because B is the box that will answer the challenge. |
| 585 | edgeB := newE2EEdge(t, bucket, false, ca.directoryURL(), e2eHost, e2eUnmintedHost, e2eLocalOnlyHost) |
| 586 | ca.validateAgainst(edgeB.serveHTTP(t)) |
| 587 | |
| 588 | // Edge A: the housekeeper. Its :80 chain is served too, so "the CA never |
| 589 | // reached the issuer" is an assertion and not an assumption. |
| 590 | edgeA := newE2EEdge(t, bucket, true, ca.directoryURL(), e2eHost, e2eUnmintedHost) |
| 591 | edgeA.serveHTTP(t) |
| 592 | |
| 593 | // B's own on-disk material from before the election moved: nothing |
| 594 | // publishes this, so only the direct local-cache last resort can serve it. |
| 595 | if err := edgeB.local.Put(context.Background(), e2eLocalOnlyHost, |
| 596 | hostCertPEM(t, e2eLocalOnlyHost, time.Now().Add(60*24*time.Hour))); err != nil { |
| 597 | t.Fatal(err) |
| 598 | } |
| 599 | |
| 600 | lnB := edgeB.serveTLS(t) |
| 601 | |
| 602 | // Before anything is minted, B has nothing for e2eHost anywhere: empty |
| 603 | // mirror, empty local cache, and autocert gated shut. The handshake must |
| 604 | // fail — otherwise the post-mint success below would prove nothing. |
| 605 | if _, err := e2eDial(lnB, e2eHost, ca.rootPool()); err == nil { |
| 606 | t.Fatal("edge B answered a handshake for a certificate that does not exist yet") |
| 607 | } |
| 608 | |
| 609 | // ---- the mint, on edge A only ---- |
| 610 | minted, err := edgeA.getCert(ecdsaHello(e2eHost)) |
| 611 | if err != nil { |
| 612 | t.Fatalf("the issuer failed to mint %s: %v", e2eHost, err) |
| 613 | } |
| 614 | ca.checkFaults(t) |
| 615 | if minted.Leaf == nil || minted.Leaf.VerifyHostname(e2eHost) != nil { |
| 616 | t.Fatalf("minted cert does not cover %s", e2eHost) |
| 617 | } |
| 618 | if err := minted.Leaf.CheckSignatureFrom(ca.root); err != nil { |
| 619 | t.Fatalf("minted cert was not issued by the CA under test: %v", err) |
| 620 | } |
| 621 | |
| 622 | t.Run("P1 the challenge was answered by the other edge", func(t *testing.T) { |
| 623 | v, ok := ca.lastValidation() |
| 624 | if !ok { |
| 625 | t.Fatal("the CA never validated anything — the order did not go through HTTP-01") |
| 626 | } |
| 627 | if !v.ok { |
| 628 | t.Fatalf("validation failed: the body served for %s was not the key authorization", v.host) |
| 629 | } |
| 630 | if v.host != e2eHost { |
| 631 | t.Fatalf("validated Host = %q, want %q", v.host, e2eHost) |
| 632 | } |
| 633 | if n := edgeB.hits.Load(); n == 0 { |
| 634 | t.Fatal("edge B's :80 chain was never reached — the challenge was not answered off-issuer") |
| 635 | } |
| 636 | if n := edgeA.hits.Load(); n != 0 { |
| 637 | t.Fatalf("edge A's :80 chain served %d request(s) — the CA must never have reached the issuer", n) |
| 638 | } |
| 639 | // The answer could only have come out of the bucket: B never ran an |
| 640 | // order, so it holds no token in memory, and its own autocert |
| 641 | // HTTPHandler 403s every challenge while gated. |
| 642 | if err := edgeB.edge.issuerHostPolicy()(context.Background(), e2eHost); !errors.Is(err, errNotIssuer) { |
| 643 | t.Fatalf("edge B's HostPolicy = %v, want errNotIssuer — its own handler must be shut", err) |
| 644 | } |
| 645 | // And it was a LIVE read, not a poll: B has never refreshed, so its |
| 646 | // snapshot is still empty — challengeResponse's read-through is the |
| 647 | // only path that could have produced the body, and the exact key it |
| 648 | // read is the order's own token entry. |
| 649 | if n := len(edgeB.mirror.snapshot().entries); n != 0 { |
| 650 | t.Fatalf("edge B's mirror holds %d entry/entries already — the read-through is not what was tested", n) |
| 651 | } |
| 652 | want := certKey(v.token + httpTokenSuffix) |
| 653 | reads, _ := edgeB.store.snapshot() |
| 654 | if !slices.Contains(reads, want) { |
| 655 | t.Fatalf("edge B read %v, want a live read of %q", reads, want) |
| 656 | } |
| 657 | }) |
| 658 | |
| 659 | accountsAfterMint, ordersAfterMint := ca.counts() |
| 660 | if accountsAfterMint != 1 || ordersAfterMint != 1 { |
| 661 | t.Fatalf("CA saw %d account(s) and %d order(s) after one mint, want 1 and 1", |
| 662 | accountsAfterMint, ordersAfterMint) |
| 663 | } |
| 664 | |
| 665 | t.Run("P2 the other edge serves the issuer's cert after one refresh", func(t *testing.T) { |
| 666 | if err := edgeB.mirror.Refresh(context.Background()); err != nil { |
| 667 | t.Fatalf("one refresh must be enough: %v", err) |
| 668 | } |
| 669 | // The in-memory mirror map holds it — the half that serves a box whose |
| 670 | // own disk is empty. |
| 671 | if c := edgeB.mirror.certificate(e2eHost, false); c == nil { |
| 672 | t.Fatal("the mirror map has no certificate for the host the issuer minted") |
| 673 | } else if !bytes.Equal(c.Leaf.Raw, minted.Leaf.Raw) { |
| 674 | t.Fatal("the mirror holds a DIFFERENT certificate than the one the issuer minted") |
| 675 | } |
| 676 | state, err := e2eDial(lnB, e2eHost, ca.rootPool()) |
| 677 | if err != nil { |
| 678 | t.Fatalf("edge B must serve the issuer's certificate: %v", err) |
| 679 | } |
| 680 | if len(state.PeerCertificates) == 0 { |
| 681 | t.Fatal("no peer certificate") |
| 682 | } |
| 683 | if !bytes.Equal(state.PeerCertificates[0].Raw, minted.Leaf.Raw) { |
| 684 | t.Fatal("edge B served a certificate that is not the one edge A minted") |
| 685 | } |
| 686 | if got := state.ServerName; got != e2eHost { |
| 687 | t.Fatalf("SNI = %q, want %q", got, e2eHost) |
| 688 | } |
| 689 | accounts, orders := ca.counts() |
| 690 | if accounts != accountsAfterMint || orders != ordersAfterMint { |
| 691 | t.Fatalf("edge B talked to the CA: accounts %d→%d, orders %d→%d — it must start zero", |
| 692 | accountsAfterMint, accounts, ordersAfterMint, orders) |
| 693 | } |
| 694 | _, wrote := edgeB.store.snapshot() |
| 695 | if len(wrote) != 0 { |
| 696 | t.Fatalf("a gated edge wrote %v to the control bucket — only the issuer may", wrote) |
| 697 | } |
| 698 | }) |
| 699 | |
| 700 | t.Run("P3 the gated twin refuses to mint and still serves", func(t *testing.T) { |
| 701 | // Edge B is edge A's twin in every respect but the gate: same |
| 702 | // construction, same store, same CA client, HTTP-01 armed. Asked for a |
| 703 | // host nobody has minted, it must refuse as the ISSUER gate and start |
| 704 | // no order at all. |
| 705 | _, ordersBefore := ca.counts() |
| 706 | _, err := edgeB.getCert(ecdsaHello(e2eUnmintedHost)) |
| 707 | if !errors.Is(err, errNotIssuer) { |
| 708 | t.Fatalf("gated twin minting %s = %v, want errNotIssuer", e2eUnmintedHost, err) |
| 709 | } |
| 710 | if _, orders := ca.counts(); orders != ordersBefore { |
| 711 | t.Fatalf("the gated twin started %d order(s)", orders-ordersBefore) |
| 712 | } |
| 713 | // ...while still serving what is on its OWN disk. autocert refuses the |
| 714 | // name before it ever looks there (HostPolicy precedes cacheGet), so |
| 715 | // this can only be the chain's direct local-cache last resort. |
| 716 | got, err := edgeB.getCert(ecdsaHello(e2eLocalOnlyHost)) |
| 717 | if err != nil { |
| 718 | t.Fatalf("a gated box must keep serving its own cached material: %v", err) |
| 719 | } |
| 720 | if got.Leaf == nil || got.Leaf.VerifyHostname(e2eLocalOnlyHost) != nil { |
| 721 | t.Fatalf("served cert does not cover %s", e2eLocalOnlyHost) |
| 722 | } |
| 723 | if _, ok := edgeB.mirror.Cert(e2eLocalOnlyHost); ok { |
| 724 | t.Fatal("test bug: that host was supposed to exist only on B's disk") |
| 725 | } |
| 726 | }) |
| 727 | |
| 728 | t.Run("P6 the mirror reads its own handle, never the fleet door", func(t *testing.T) { |
| 729 | // Structural: the mirror holds the edge's own deployment-store handle. |
| 730 | if edgeB.mirror.s != release.Store(edgeB.store) { |
| 731 | t.Fatal("the mirror is not reading through this box's own store handle") |
| 732 | } |
| 733 | if edgeB.store == edgeA.store { |
| 734 | t.Fatal("test bug: the two edges must hold separate handles on the one bucket") |
| 735 | } |
| 736 | // And the door could not have served those reads even if something |
| 737 | // tried: the fleet ACL (storedoor.go) refuses every key under |
| 738 | // control/certs/ for every tenant and fleet, so a fleet box reaching |
| 739 | // for cert material over the door gets the byte-identical 404. |
| 740 | for _, key := range []string{certKey(e2eHost), certKey("acme_account+key"), certPrefix} { |
| 741 | if aclRead("bdk", "pi-fleet", key) { |
| 742 | t.Fatalf("the fleet door ACL allows %q — cert material must never be reachable through it", key) |
| 743 | } |
| 744 | } |
| 745 | // Every read this box's mirror made stayed inside the one prefix. |
| 746 | reads, _ := edgeB.store.snapshot() |
| 747 | for _, key := range reads { |
| 748 | if !strings.HasPrefix(key, certPrefix) { |
| 749 | t.Fatalf("the mirror read %q, outside %s", key, certPrefix) |
| 750 | } |
| 751 | } |
| 752 | }) |
| 753 | |
| 754 | ca.checkFaults(t) |
| 755 | } |
| 756 | |
| 757 | // ---- the CA's remaining handlers --------------------------------------------- |
| 758 | // |
| 759 | // Split out below the scenario they serve, so the test reads first. |
| 760 | |
| 761 | func (ca *fakeACME) orderID(r *http.Request) (string, *fakeACMEOrder, bool) { |
| 762 | idx := strings.LastIndex(r.URL.Path, "/") |
| 763 | id := r.URL.Path[idx+1:] |
| 764 | ca.mu.Lock() |
| 765 | defer ca.mu.Unlock() |
| 766 | o, ok := ca.byID[id] |
| 767 | return id, o, ok |
| 768 | } |
| 769 | |
| 770 | // serveAuthorization answers POST-as-GET on an authorization, offering the one |
| 771 | // http-01 challenge (see fakeACME's doc for why only one type). |
| 772 | func (ca *fakeACME) serveAuthorization(w http.ResponseWriter, r *http.Request) { |
| 773 | id, o, ok := ca.orderID(r) |
| 774 | if !ok { |
| 775 | ca.recordFault("authz %s: unknown", id) |
| 776 | http.Error(w, "no such authz", http.StatusNotFound) |
| 777 | return |
| 778 | } |
| 779 | ca.mu.Lock() |
| 780 | body := map[string]any{ |
| 781 | "identifier": map[string]string{"type": "dns", "value": o.domain}, |
| 782 | "status": o.authz, |
| 783 | "challenges": []map[string]string{{ |
| 784 | "type": "http-01", |
| 785 | "url": ca.url("/chal/" + id), |
| 786 | "token": o.token, |
| 787 | "status": o.authz, |
| 788 | }}, |
| 789 | } |
| 790 | ca.mu.Unlock() |
| 791 | writeJSON(w, http.StatusOK, body) |
| 792 | } |
| 793 | |
| 794 | // serveChallengeAccept is the CA's validation. It runs synchronously: the |
| 795 | // whole point of the read-through in challengeResponse is that a validation's |
| 796 | // life is measured in seconds, and a CA that answered on the next poll would |
| 797 | // be testing the wrong thing. |
| 798 | func (ca *fakeACME) serveChallengeAccept(w http.ResponseWriter, r *http.Request) { |
| 799 | id, o, ok := ca.orderID(r) |
| 800 | if !ok { |
| 801 | ca.recordFault("challenge %s: unknown", id) |
| 802 | http.Error(w, "no such challenge", http.StatusNotFound) |
| 803 | return |
| 804 | } |
| 805 | ca.mu.Lock() |
| 806 | base, thumb, domain, token := ca.validateVia, ca.thumb, o.domain, o.token |
| 807 | ca.mu.Unlock() |
| 808 | |
| 809 | url := base + acmeChallengePrefix + token |
| 810 | got, ferr := ca.fetchChallenge(url, domain) |
| 811 | want := token + "." + thumb |
| 812 | valid := ferr == nil && got == want |
| 813 | |
| 814 | ca.mu.Lock() |
| 815 | ca.validations = append(ca.validations, fakeACMEValidation{url: url, host: domain, token: token, ok: valid}) |
| 816 | if valid { |
| 817 | o.authz, o.status = "valid", "ready" |
| 818 | } else { |
| 819 | o.authz, o.status = "invalid", "invalid" |
| 820 | ca.faults = append(ca.faults, fmt.Sprintf( |
| 821 | "http-01 for %s at %s: body %q, want %q (fetch err %v)", domain, url, got, want, ferr)) |
| 822 | } |
| 823 | ca.mu.Unlock() |
| 824 | |
| 825 | writeJSON(w, http.StatusOK, map[string]string{ |
| 826 | "type": "http-01", "url": ca.url("/chal/" + id), "token": token, "status": "valid", |
| 827 | }) |
| 828 | } |
| 829 | |
| 830 | // fetchChallenge is the validator: an ordinary HTTP GET carrying the Host |
| 831 | // header of the name under validation, exactly as a CA sends it (and exactly |
| 832 | // what challengeResponse's host bound is calibrated against). |
| 833 | func (ca *fakeACME) fetchChallenge(url, host string) (string, error) { |
| 834 | req, err := http.NewRequest("GET", url, nil) |
| 835 | if err != nil { |
| 836 | return "", err |
| 837 | } |
| 838 | req.Host = host |
| 839 | client := &http.Client{Timeout: 10 * time.Second} |
| 840 | res, err := client.Do(req) |
| 841 | if err != nil { |
| 842 | return "", err |
| 843 | } |
| 844 | defer res.Body.Close() |
| 845 | body, err := io.ReadAll(io.LimitReader(res.Body, 1<<16)) |
| 846 | if err != nil { |
| 847 | return "", err |
| 848 | } |
| 849 | if res.StatusCode != http.StatusOK { |
| 850 | return "", fmt.Errorf("challenge fetch: %s", res.Status) |
| 851 | } |
| 852 | return string(body), nil |
| 853 | } |
| 854 | |
| 855 | func (ca *fakeACME) serveOrder(w http.ResponseWriter, r *http.Request) { |
| 856 | id, _, ok := ca.orderID(r) |
| 857 | if !ok { |
| 858 | ca.recordFault("order %s: unknown", id) |
| 859 | http.Error(w, "no such order", http.StatusNotFound) |
| 860 | return |
| 861 | } |
| 862 | w.Header().Set("Location", ca.url("/order/"+id)) |
| 863 | writeJSON(w, http.StatusOK, ca.orderJSON(id)) |
| 864 | } |
| 865 | |
| 866 | func (ca *fakeACME) serveFinalize(w http.ResponseWriter, r *http.Request) { |
| 867 | id, o, ok := ca.orderID(r) |
| 868 | if !ok { |
| 869 | ca.recordFault("finalize %s: unknown", id) |
| 870 | http.Error(w, "no such order", http.StatusNotFound) |
| 871 | return |
| 872 | } |
| 873 | _, payload, err := decodeJWS(r) |
| 874 | if err != nil { |
| 875 | ca.recordFault("finalize %s: %v", id, err) |
| 876 | http.Error(w, "bad jws", http.StatusBadRequest) |
| 877 | return |
| 878 | } |
| 879 | var req struct { |
| 880 | CSR string `json:"csr"` |
| 881 | } |
| 882 | if err := json.Unmarshal(payload, &req); err != nil { |
| 883 | ca.recordFault("finalize %s: %v", id, err) |
| 884 | http.Error(w, "bad finalize", http.StatusBadRequest) |
| 885 | return |
| 886 | } |
| 887 | der, err := base64.RawURLEncoding.DecodeString(req.CSR) |
| 888 | if err != nil { |
| 889 | ca.recordFault("finalize %s: csr: %v", id, err) |
| 890 | http.Error(w, "bad csr", http.StatusBadRequest) |
| 891 | return |
| 892 | } |
| 893 | ca.mu.Lock() |
| 894 | domain, authz := o.domain, o.authz |
| 895 | ca.mu.Unlock() |
| 896 | if authz != "valid" { |
| 897 | ca.recordFault("finalize %s: authorization is %q — a CA must not issue on an unvalidated order", id, authz) |
| 898 | http.Error(w, "not authorized", http.StatusForbidden) |
| 899 | return |
| 900 | } |
| 901 | chain, err := ca.issue(der, domain) |
| 902 | if err != nil { |
| 903 | ca.recordFault("finalize %s: %v", id, err) |
| 904 | http.Error(w, "cannot issue", http.StatusBadRequest) |
| 905 | return |
| 906 | } |
| 907 | ca.mu.Lock() |
| 908 | o.chain, o.status = chain, "valid" |
| 909 | ca.mu.Unlock() |
| 910 | w.Header().Set("Location", ca.url("/order/"+id)) |
| 911 | writeJSON(w, http.StatusOK, ca.orderJSON(id)) |
| 912 | } |
| 913 | |
| 914 | func (ca *fakeACME) serveCert(w http.ResponseWriter, r *http.Request) { |
| 915 | id, o, ok := ca.orderID(r) |
| 916 | if !ok { |
| 917 | ca.recordFault("cert %s: unknown", id) |
| 918 | http.Error(w, "no such cert", http.StatusNotFound) |
| 919 | return |
| 920 | } |
| 921 | ca.mu.Lock() |
| 922 | chain := o.chain |
| 923 | ca.mu.Unlock() |
| 924 | w.Header().Set("Content-Type", "application/pem-certificate-chain") |
| 925 | w.WriteHeader(http.StatusOK) |
| 926 | _, _ = w.Write(chain) |
| 927 | } |
| 928 | |
| 929 | // issue signs the CSR's public key for domain under the fake root. The CSR's |
| 930 | // own signature and name are checked: a CA that issued for whatever it was |
| 931 | // asked would let a chain-of-custody bug through unnoticed. |
| 932 | func (ca *fakeACME) issue(csrDER []byte, domain string) ([]byte, error) { |
| 933 | csr, err := x509.ParseCertificateRequest(csrDER) |
| 934 | if err != nil { |
| 935 | return nil, fmt.Errorf("parse csr: %w", err) |
| 936 | } |
| 937 | if err := csr.CheckSignature(); err != nil { |
| 938 | return nil, fmt.Errorf("csr signature: %w", err) |
| 939 | } |
| 940 | if len(csr.DNSNames) != 1 || csr.DNSNames[0] != domain { |
| 941 | return nil, fmt.Errorf("csr names %v, want [%s]", csr.DNSNames, domain) |
| 942 | } |
| 943 | serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 96)) |
| 944 | if err != nil { |
| 945 | return nil, err |
| 946 | } |
| 947 | tmpl := &x509.Certificate{ |
| 948 | SerialNumber: serial, |
| 949 | Subject: pkix.Name{CommonName: domain}, |
| 950 | DNSNames: []string{domain}, |
| 951 | NotBefore: time.Now().Add(-time.Hour), |
| 952 | // 90 days, like the real thing: autocert's renewal timer is then |
| 953 | // ~60 days out and nothing fires during the test. |
| 954 | NotAfter: time.Now().Add(90 * 24 * time.Hour), |
| 955 | KeyUsage: x509.KeyUsageDigitalSignature, |
| 956 | ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, |
| 957 | } |
| 958 | der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.root, csr.PublicKey, ca.rootKey) |
| 959 | if err != nil { |
| 960 | return nil, err |
| 961 | } |
| 962 | var buf bytes.Buffer |
| 963 | if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: der}); err != nil { |
| 964 | return nil, err |
| 965 | } |
| 966 | if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: ca.root.Raw}); err != nil { |
| 967 | return nil, err |
| 968 | } |
| 969 | return buf.Bytes(), nil |
| 970 | } |
| 971 | |
| 972 | // ---- P4: the renewal timer's cache-first check ------------------------------- |
| 973 | |
| 974 | // R3: autocert's renewal timers never consult HostPolicy (see |
| 975 | // issuerHostPolicy), so a box that has stood down still has armed timers. What |
| 976 | // stops one ordering is renewal.go's do(), which reads the CACHE first and |
| 977 | // returns without touching a CA when the cached cert is far enough from |
| 978 | // expiry. The mirror's syncLocal is what puts a fresher cert there. |
| 979 | // |
| 980 | // The boundary, stated rather than papered over: autocert's real timer is not |
| 981 | // fast-forwarded here — time.AfterFunc's deadline is unreachable from a test |
| 982 | // and faking it would test the fake. What IS proven is every input the timer's |
| 983 | // decision reads: the local cache holds the fresher bytes after one Refresh, |
| 984 | // autocert's own cacheGet (the exact call do() makes, reached here through |
| 985 | // GetCertificate) returns the fresher certificate, and the renewal threshold |
| 986 | // re-derived from renewal.go's next() flips from "due" to "not due" across the |
| 987 | // refresh. |
| 988 | func TestCertSyncE2EFresherBucketCertReachesAutocertsCache(t *testing.T) { |
| 989 | e2eQuietLogs(t) |
| 990 | ctx := context.Background() |
| 991 | store := release.NewDirStore(t.TempDir()) |
| 992 | local := autocert.DirCache(t.TempDir()) |
| 993 | |
| 994 | // Stale but live: a 90-day cert with 20 days left, which is inside |
| 995 | // autocert's 30-day renewal threshold — this box's timer is due to order. |
| 996 | stale := e2eDatedCertPEM(t, e2eHost, time.Now().Add(-70*24*time.Hour), time.Now().Add(20*24*time.Hour)) |
| 997 | if err := local.Put(ctx, e2eHost, stale); err != nil { |
| 998 | t.Fatal(err) |
| 999 | } |
| 1000 | if due := e2eRenewalDue(t, stale); !due { |
| 1001 | t.Fatal("test bug: the seeded cert must be inside the renewal threshold, or there is nothing to rescue") |
| 1002 | } |
| 1003 | |
| 1004 | // The issuer has already renewed and published: 90 days, all of them left. |
| 1005 | fresh := e2eDatedCertPEM(t, e2eHost, time.Now().Add(-time.Hour), time.Now().Add(90*24*time.Hour)) |
| 1006 | putBucket(t, store, e2eHost, fresh) |
| 1007 | |
| 1008 | m := newCertMirror(store, local) |
| 1009 | if err := m.Refresh(ctx); err != nil { |
| 1010 | t.Fatal(err) |
| 1011 | } |
| 1012 | |
| 1013 | got, err := localCert(t, local, e2eHost) |
| 1014 | if err != nil { |
| 1015 | t.Fatalf("the mirror must write fresh material into this box's own cache: %v", err) |
| 1016 | } |
| 1017 | if !bytes.Equal(got, fresh) { |
| 1018 | t.Fatal("the local cache still holds the stale cert — the renewal timer would go to the CA") |
| 1019 | } |
| 1020 | if due := e2eRenewalDue(t, got); due { |
| 1021 | t.Fatal("the cached cert is still inside the renewal threshold: do() would fall through and order") |
| 1022 | } |
| 1023 | |
| 1024 | // The cache-first read do() makes, made through the public door onto it. |
| 1025 | // The manager is UNGATED (this box may have won the election back) and |
| 1026 | // points at a dead directory, so a cache miss fails offline and fast |
| 1027 | // rather than reaching a CA. |
| 1028 | e := testIssuerEdge(e2eHost) |
| 1029 | served, err := offlineManager(t, local, e.issuerHostPolicy()).GetCertificate(ecdsaHello(e2eHost)) |
| 1030 | if err != nil { |
| 1031 | t.Fatalf("autocert's own cache read must find the mirrored cert: %v", err) |
| 1032 | } |
| 1033 | freshLeaf := e2eLeafOf(t, fresh) |
| 1034 | staleLeaf := e2eLeafOf(t, stale) |
| 1035 | if !freshLeaf.NotAfter.After(staleLeaf.NotAfter) { |
| 1036 | t.Fatal("test bug: the two certs are not ordered by expiry") |
| 1037 | } |
| 1038 | if !served.Leaf.NotAfter.Equal(freshLeaf.NotAfter) { |
| 1039 | t.Fatalf("autocert served a cert expiring %s, want the mirrored one at %s", |
| 1040 | served.Leaf.NotAfter, freshLeaf.NotAfter) |
| 1041 | } |
| 1042 | } |
| 1043 | |
| 1044 | // e2eDatedCertPEM is hostCertPEM with a caller-chosen NotBefore, so a test can |
| 1045 | // build a cert whose LIFETIME (and therefore autocert's renewal threshold) is |
| 1046 | // realistic rather than an artefact of the helper. |
| 1047 | func e2eDatedCertPEM(t *testing.T, host string, notBefore, notAfter time.Time) []byte { |
| 1048 | t.Helper() |
| 1049 | key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) |
| 1050 | if err != nil { |
| 1051 | t.Fatal(err) |
| 1052 | } |
| 1053 | tmpl := &x509.Certificate{ |
| 1054 | SerialNumber: big.NewInt(2), |
| 1055 | Subject: pkix.Name{CommonName: host}, |
| 1056 | DNSNames: []string{host}, |
| 1057 | NotBefore: notBefore, |
| 1058 | NotAfter: notAfter, |
| 1059 | } |
| 1060 | der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) |
| 1061 | if err != nil { |
| 1062 | t.Fatal(err) |
| 1063 | } |
| 1064 | leaf, err := x509.ParseCertificate(der) |
| 1065 | if err != nil { |
| 1066 | t.Fatal(err) |
| 1067 | } |
| 1068 | raw, err := encodeWildcardPEM(&tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: leaf}, key) |
| 1069 | if err != nil { |
| 1070 | t.Fatal(err) |
| 1071 | } |
| 1072 | return raw |
| 1073 | } |
| 1074 | |
| 1075 | func e2eLeafOf(t *testing.T, entry []byte) *x509.Certificate { |
| 1076 | t.Helper() |
| 1077 | cert, err := parseCertEntry(entry) |
| 1078 | if err != nil { |
| 1079 | t.Fatal(err) |
| 1080 | } |
| 1081 | return cert.Leaf |
| 1082 | } |
| 1083 | |
| 1084 | // e2eRenewalDue re-derives autocert's own renewal decision (renewal.go's |
| 1085 | // next(), with no RenewBefore set): threshold = min(lifetime/3, 30 days), and |
| 1086 | // the timer's cache-first branch falls through to an ORDER when the renewal |
| 1087 | // moment has already passed. Jitter is ignored — it is at most an hour and |
| 1088 | // both fixtures here are days from the boundary. |
| 1089 | func e2eRenewalDue(t *testing.T, entry []byte) bool { |
| 1090 | t.Helper() |
| 1091 | leaf := e2eLeafOf(t, entry) |
| 1092 | threshold := min(leaf.NotAfter.Sub(leaf.NotBefore)/3, 30*24*time.Hour) |
| 1093 | return !leaf.NotAfter.Add(-threshold).After(time.Now()) |
| 1094 | } |
| 1095 | |
| 1096 | // ---- P5: Refresh under concurrency ------------------------------------------- |
| 1097 | |
| 1098 | // The mirror has no refreshMu: Refresh is written for the single poll loop |
| 1099 | // that calls it (run(), plus the one synchronous boot pass edge.go runs BEFORE |
| 1100 | // that goroutine starts). This test runs it from several goroutines at once |
| 1101 | // anyway, because "only one caller today" is a property of the callers and not |
| 1102 | // of the code, and the -race gate is where a wrong answer to that question is |
| 1103 | // cheap to learn. |
| 1104 | // |
| 1105 | // What it asserts is memory safety and a coherent OUTCOME: whichever |
| 1106 | // interleaving wins, the published snapshot and the local cache both end up |
| 1107 | // holding every entry the bucket holds. It deliberately does NOT assert |
| 1108 | // linearizability — two overlapping polls can publish out of order, and the |
| 1109 | // next poll converges. See the task report for that finding. |
| 1110 | func TestCertSyncE2EConcurrentRefreshIsRaceFree(t *testing.T) { |
| 1111 | e2eQuietLogs(t) |
| 1112 | ctx := context.Background() |
| 1113 | store := release.NewDirStore(t.TempDir()) |
| 1114 | hosts := []string{"a." + e2eHost, "b." + e2eHost, "c." + e2eHost} |
| 1115 | want := map[string][]byte{} |
| 1116 | for _, h := range hosts { |
| 1117 | body := hostCertPEM(t, h, time.Now().Add(80*24*time.Hour)) |
| 1118 | putBucket(t, store, h, body) |
| 1119 | want[h] = body |
| 1120 | } |
| 1121 | putBucket(t, store, "acme_account+key", []byte("ACCOUNT KEY")) |
| 1122 | |
| 1123 | local := autocert.DirCache(t.TempDir()) |
| 1124 | m := newCertMirror(store, local) |
| 1125 | |
| 1126 | var wg sync.WaitGroup |
| 1127 | for range 4 { |
| 1128 | wg.Add(1) |
| 1129 | go func() { |
| 1130 | defer wg.Done() |
| 1131 | for range 5 { |
| 1132 | if err := m.Refresh(ctx); err != nil { |
| 1133 | t.Errorf("concurrent refresh: %v", err) |
| 1134 | return |
| 1135 | } |
| 1136 | } |
| 1137 | }() |
| 1138 | } |
| 1139 | wg.Wait() |
| 1140 | |
| 1141 | snap := m.snapshot() |
| 1142 | for h, body := range want { |
| 1143 | if got, ok := snap.entries[h]; !ok || !bytes.Equal(got, body) { |
| 1144 | t.Fatalf("snapshot entry %s = %q (%v), want the published body", h, got, ok) |
| 1145 | } |
| 1146 | if snap.certs[h] == nil { |
| 1147 | t.Fatalf("snapshot has no parsed certificate for %s", h) |
| 1148 | } |
| 1149 | got, err := localCert(t, local, h) |
| 1150 | if err != nil || !bytes.Equal(got, body) { |
| 1151 | t.Fatalf("local cache %s = %q, %v — every entry must land exactly once", h, got, err) |
| 1152 | } |
| 1153 | } |
| 1154 | if got, err := localCert(t, local, "acme_account+key"); err != nil || string(got) != "ACCOUNT KEY" { |
| 1155 | t.Fatalf("account key seeding raced: %q, %v", got, err) |
| 1156 | } |
| 1157 | } |
| 1158 | |
| 1159 | // ---- #277: the non-home edge answers, and the home-boxed name mints --------- |
| 1160 | |
| 1161 | // e2eRemoteHomedHost is homed on edge A and absent from edge B's registry — |
| 1162 | // the shape the original scenario deliberately did not have (there, B carried |
| 1163 | // every host in its own route table). |
| 1164 | const e2eRemoteHomedHost = "remote.bab.example.test" |
| 1165 | |
| 1166 | // The automated shadow of the dev proof that failed as #277: DNS for an |
| 1167 | // exactly-named registry ROUTE points at edge B, which homes nothing for that |
| 1168 | // name. B backhauls to A, A (the issuer) sees the hello and orders, and the CA |
| 1169 | // then validates against B — the box whose registry has never heard of the |
| 1170 | // host. Before the widening, B refused the challenge, every authorization |
| 1171 | // failed, and the name's Let's Encrypt budget went in minutes. After it, B |
| 1172 | // answers out of the bucket, A mints, and the certificate reaches B's mirror. |
| 1173 | // |
| 1174 | // A route, deliberately, and not an attached custom domain: an alias attach |
| 1175 | // lands in route_aliases (internal/domains/aliases.go), which edgesFilePass |
| 1176 | // never publishes — so no alias is in any home map, on any box. The spec |
| 1177 | // amendment records that as an open follow-up rather than a claim this test |
| 1178 | // could make. |
| 1179 | // |
| 1180 | // The backhaul leg itself is not re-proved here (homemap_test.go owns it); |
| 1181 | // what this pins is that everything CERT-side works with the host in ONE |
| 1182 | // box's registry and the other box's home map. |
| 1183 | func TestNonHomeEdgeAnswersValidationForRemoteHomedHost(t *testing.T) { |
| 1184 | e2eQuietLogs(t) |
| 1185 | bucket := t.TempDir() |
| 1186 | ca := newFakeACME(t) |
| 1187 | |
| 1188 | // Edge B: no routes at all for the name — this is the box DNS points at. |
| 1189 | edgeB := newE2EEdge(t, bucket, false, ca.directoryURL()) |
| 1190 | ca.validateAgainst(edgeB.serveHTTP(t)) |
| 1191 | if edgeB.edge.Allowed(e2eRemoteHomedHost) { |
| 1192 | t.Fatal("test bug: edge B must not carry the host in its own registry") |
| 1193 | } |
| 1194 | |
| 1195 | // ...but it mirrors the deployment's edges files, in which edge A homes |
| 1196 | // the name exactly. That, and only that, is what admits the validation. |
| 1197 | homes := newHomeMap(release.NewDirStore(bucket), "edge-b") |
| 1198 | homedOn(t, homes, "edge-a", "203.0.113.10", e2eRemoteHomedHost, "*.other.example.test") |
| 1199 | edgeB.edge.homes.Store(homes) |
| 1200 | |
| 1201 | // Edge A: the housekeeper, and the box that homes the route. Its own :80 |
| 1202 | // chain is served purely so "the CA never reached the issuer" is an |
| 1203 | // assertion rather than an assumption. |
| 1204 | edgeA := newE2EEdge(t, bucket, true, ca.directoryURL(), e2eRemoteHomedHost) |
| 1205 | edgeA.serveHTTP(t) |
| 1206 | |
| 1207 | lnB := edgeB.serveTLS(t) |
| 1208 | if _, err := e2eDial(lnB, e2eRemoteHomedHost, ca.rootPool()); err == nil { |
| 1209 | t.Fatal("edge B answered a handshake for a certificate that does not exist yet") |
| 1210 | } |
| 1211 | |
| 1212 | // The mint, on edge A — exactly what a backhauled hello triggers there. |
| 1213 | minted, err := edgeA.getCert(ecdsaHello(e2eRemoteHomedHost)) |
| 1214 | if err != nil { |
| 1215 | t.Fatalf("the issuer failed to mint a host it homes: %v", err) |
| 1216 | } |
| 1217 | ca.checkFaults(t) |
| 1218 | if minted.Leaf == nil || minted.Leaf.VerifyHostname(e2eRemoteHomedHost) != nil { |
| 1219 | t.Fatalf("minted cert does not cover %s", e2eRemoteHomedHost) |
| 1220 | } |
| 1221 | |
| 1222 | t.Run("the validation was answered by the box that homes nothing", func(t *testing.T) { |
| 1223 | v, ok := ca.lastValidation() |
| 1224 | if !ok { |
| 1225 | t.Fatal("the CA never validated anything") |
| 1226 | } |
| 1227 | if !v.ok || v.host != e2eRemoteHomedHost { |
| 1228 | t.Fatalf("validation of %q ok=%v — the body B served was not the key authorization", v.host, v.ok) |
| 1229 | } |
| 1230 | if n := edgeB.hits.Load(); n == 0 { |
| 1231 | t.Fatal("edge B's :80 chain was never reached") |
| 1232 | } |
| 1233 | if n := edgeA.hits.Load(); n != 0 { |
| 1234 | t.Fatalf("edge A's :80 chain served %d request(s) — the CA must never have reached the issuer", n) |
| 1235 | } |
| 1236 | // It came out of the bucket, live: B never polled, and its own |
| 1237 | // autocert handler is shut for this name by the issuer gate. |
| 1238 | if n := len(edgeB.mirror.snapshot().entries); n != 0 { |
| 1239 | t.Fatalf("edge B's mirror holds %d entry/entries — the read-through is not what was tested", n) |
| 1240 | } |
| 1241 | if err := edgeB.edge.issuerHostPolicy()(context.Background(), e2eRemoteHomedHost); !errors.Is(err, errNotIssuer) { |
| 1242 | t.Fatalf("edge B's HostPolicy = %v, want errNotIssuer — a non-issuer orders nothing, however it is named", err) |
| 1243 | } |
| 1244 | want := certKey(v.token + httpTokenSuffix) |
| 1245 | reads, _ := edgeB.store.snapshot() |
| 1246 | if !slices.Contains(reads, want) { |
| 1247 | t.Fatalf("edge B read %v, want a live read of %q", reads, want) |
| 1248 | } |
| 1249 | }) |
| 1250 | |
| 1251 | t.Run("the tls-alpn-01 twin is admitted for the same name", func(t *testing.T) { |
| 1252 | // The fake CA offers http-01 only (its doc says why), so the ALPN half |
| 1253 | // is proved at the same seam the unit tests use: a token cert in the |
| 1254 | // bucket, an acme-tls/1 hello at edge B for a name only edge A homes. |
| 1255 | // Written through a THIRD handle on the bucket, standing in for the |
| 1256 | // issuer's publishingCache: edge B's own handle must stay write-free, |
| 1257 | // which the last subtest asserts. |
| 1258 | issuerHandle := release.NewDirStore(bucket) |
| 1259 | putBucket(t, issuerHandle, e2eRemoteHomedHost+alpnTokenSuffix, |
| 1260 | hostCertPEM(t, e2eRemoteHomedHost, time.Now().Add(24*time.Hour))) |
| 1261 | if c := edgeB.mirror.alpnTokenCert(context.Background(), e2eRemoteHomedHost); c == nil { |
| 1262 | t.Fatal("edge B refused a tls-alpn-01 validation for a host edge A homes") |
| 1263 | } |
| 1264 | // And the label under the mirrored catch-all is still refused (#180). |
| 1265 | // The token cert for it is PUT FIRST and the label is asserted |
| 1266 | // dialable: without both, a nil answer here would prove only that the |
| 1267 | // bucket is empty, and the refusal would pass even if KnownHost leaked |
| 1268 | // through Home()'s one-label rule. |
| 1269 | const invented = "invented.other.example.test" |
| 1270 | putBucket(t, issuerHandle, invented+alpnTokenSuffix, |
| 1271 | hostCertPEM(t, invented, time.Now().Add(24*time.Hour))) |
| 1272 | if _, _, ok := edgeB.edge.homes.Load().Home(invented); !ok { |
| 1273 | t.Fatal("test bug: the invented label is supposed to be DIALABLE through the one-label rule") |
| 1274 | } |
| 1275 | if c := edgeB.mirror.alpnTokenCert(context.Background(), invented); c != nil { |
| 1276 | t.Fatal("a label under a mirrored catch-all was served a token cert") |
| 1277 | } |
| 1278 | }) |
| 1279 | |
| 1280 | t.Run("the minted cert reaches the non-home edge", func(t *testing.T) { |
| 1281 | if err := edgeB.mirror.Refresh(context.Background()); err != nil { |
| 1282 | t.Fatalf("one refresh must be enough: %v", err) |
| 1283 | } |
| 1284 | state, err := e2eDial(lnB, e2eRemoteHomedHost, ca.rootPool()) |
| 1285 | if err != nil { |
| 1286 | t.Fatalf("edge B must serve the certificate edge A minted: %v", err) |
| 1287 | } |
| 1288 | if len(state.PeerCertificates) == 0 || !bytes.Equal(state.PeerCertificates[0].Raw, minted.Leaf.Raw) { |
| 1289 | t.Fatal("edge B served a certificate that is not the one edge A minted") |
| 1290 | } |
| 1291 | _, wrote := edgeB.store.snapshot() |
| 1292 | if len(wrote) != 0 { |
| 1293 | t.Fatalf("a gated edge wrote %v to the control bucket — only the issuer may", wrote) |
| 1294 | } |
| 1295 | }) |
| 1296 | |
| 1297 | ca.checkFaults(t) |
| 1298 | } |
| 1299 | |
amadan