| 1 | package edge |
| 2 | |
| 3 | // Task 6 of the multi-region backhaul plan: the acceptance test the whole of |
| 4 | // Part 3 exists for — ONE control bucket, TWO edges, and a request that lands |
| 5 | // in the wrong region and is served anyway. |
| 6 | // |
| 7 | // Edge A ("annie") homes app.backhaul.test and runs the tenant's app behind it. |
| 8 | // Edge B ("betty") has never heard of that host: no registry route, no |
| 9 | // certificate of its own, no way to reach the app except through A. Everything |
| 10 | // between the two is the bucket — the per-box edges files (Task 1's schema, |
| 11 | // written here directly because the publishing loop lives in internal/host), |
| 12 | // the mesh key (Task 2), the certs Part 2 distributes — and one TLS hop. |
| 13 | // |
| 14 | // What makes this an e2e rather than a bigger unit test is that BOTH doors are |
| 15 | // real and neither is stubbed: |
| 16 | // |
| 17 | // - the client completes a real TLS handshake with the SENDING edge, with a |
| 18 | // real x509 verification against the test CA, for a host that edge does not |
| 19 | // route. That is the invariant the whole feature rides on and the one the |
| 20 | // unit tests can only assert one branch of at a time: certMirror must |
| 21 | // answer a hello for a host this box has no route for, or the client never |
| 22 | // gets far enough to be backhauled at all (P1 below); |
| 23 | // - the hop itself is a second real TLS handshake, from B's own transport to |
| 24 | // A's front door, verified the same way. |
| 25 | // |
| 26 | // The two edges are wired exactly as RunWithActivatorReady wires one (raw local |
| 27 | // cache kept aside, publishing wrapper over it, mirror built from the RAW |
| 28 | // handle, autocert behind the mirror, the :80 chain composed in the same |
| 29 | // order), and every poll is DRIVEN — Refresh is called, never waited for. There |
| 30 | // are no sleeps in this file. |
| 31 | // |
| 32 | // Neither edge is the deployment's certificate issuer. That is deliberate and |
| 33 | // it is the sharp case, not a convenience: issuance is Part 2's subject, while |
| 34 | // what Part 3 needs is that a box which may never talk to a CA can still answer |
| 35 | // handshakes for hosts it does not serve. Both managers point at a dead ACME |
| 36 | // directory as well, so nothing here can reach a CA even by accident. |
| 37 | // |
| 38 | // Conventions are the neighbouring harnesses' (certsync_e2e_test.go for the |
| 39 | // two-edge shape and the e2e* naming, homemap_test.go for the hop vocabulary |
| 40 | // helpers): real loopback listeners, helpers reused rather than re-cut, bounds |
| 41 | // asserted rather than narrated. No t.Parallel anywhere — this package's tests |
| 42 | // swap package-level seams (backhaulPort, backhaulRootCAs) and the log writer. |
| 43 | |
| 44 | import ( |
| 45 | "context" |
| 46 | "crypto/ecdsa" |
| 47 | "crypto/elliptic" |
| 48 | "crypto/rand" |
| 49 | "crypto/tls" |
| 50 | "crypto/x509" |
| 51 | "crypto/x509/pkix" |
| 52 | "fmt" |
| 53 | "io" |
| 54 | "math/big" |
| 55 | "net" |
| 56 | "net/http" |
| 57 | "net/http/httptest" |
| 58 | "strings" |
| 59 | "sync/atomic" |
| 60 | "testing" |
| 61 | "time" |
| 62 | |
| 63 | "golang.org/x/crypto/acme/autocert" |
| 64 | |
| 65 | "github.com/carlosframework/platform/internal/geoip" |
| 66 | "github.com/carlosframework/platform/internal/registry" |
| 67 | "github.com/carlosframework/platform/internal/release" |
| 68 | ) |
| 69 | |
| 70 | const ( |
| 71 | // e2eHomedHost is the host the whole scenario turns on: routed on edge A, |
| 72 | // unknown to edge B, homed on A in every box's map. |
| 73 | e2eHomedHost = "app.backhaul.test" |
| 74 | // e2eMovedHost is homed on A in the MAP but no longer routed there — the |
| 75 | // stale-map case, which is what a placement move looks like from the |
| 76 | // sending edge for one publish cycle. |
| 77 | e2eMovedHost = "moved.backhaul.test" |
| 78 | // e2eDeadHost is homed on a box that answers nothing: the poisoned map. |
| 79 | e2eDeadHost = "dead.backhaul.test" |
| 80 | // e2eUnhomedHost is in nobody's edges file and nobody's registry — the |
| 81 | // answer for it must stay the plain unknown-host 404 on every box. |
| 82 | e2eUnhomedHost = "nothere.backhaul.test" |
| 83 | // e2eSenderHost is edge B's OWN route, so B is a real box with traffic of |
| 84 | // its own rather than a bare forwarder. |
| 85 | e2eSenderHost = "local.betty.test" |
| 86 | |
| 87 | e2eHomeMarker = "served by the home edge's own app" |
| 88 | e2eSenderMarker = "served by the sending edge's own app" |
| 89 | ) |
| 90 | |
| 91 | // ---- the tenant app ---------------------------------------------------------- |
| 92 | |
| 93 | // e2eApp is a tenant's app behind an edge: a marker body that says WHICH box's |
| 94 | // upstream produced it, plus the trust-bearing headers it was handed. One |
| 95 | // response therefore proves both halves at once — that the bytes crossed the |
| 96 | // hop, and what the app was told about who called. |
| 97 | // |
| 98 | // X-Forwarded-For is rendered from Values(), joined visibly, because the |
| 99 | // property under test is the LENGTH of the chain as much as its content: a |
| 100 | // backhaul that appended the sending edge would show up here as a second |
| 101 | // element (or a second header line), and Get() would hide it. |
| 102 | func e2eApp(t *testing.T, marker string) *httptest.Server { |
| 103 | t.Helper() |
| 104 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 105 | fmt.Fprintf(w, "%s|xff=%s|xfh=%s|hop=%s|auth=%s|geo=%s|hopgeo=%s", marker, |
| 106 | strings.Join(r.Header.Values("X-Forwarded-For"), " + "), |
| 107 | r.Header.Get("X-Forwarded-Host"), |
| 108 | r.Header.Get(hopHeader), r.Header.Get(hopAuthHeader), |
| 109 | r.Header.Get(geoip.HeaderCountry), r.Header.Get(geoip.HeaderHop)) |
| 110 | })) |
| 111 | t.Cleanup(ts.Close) |
| 112 | return ts |
| 113 | } |
| 114 | |
| 115 | // e2eSeen is one parsed e2eApp response. |
| 116 | type e2eSeen struct { |
| 117 | marker, xff, xfh, hop, auth string |
| 118 | geoCountry, hopGeo string |
| 119 | } |
| 120 | |
| 121 | func e2eParseApp(t *testing.T, body string) e2eSeen { |
| 122 | t.Helper() |
| 123 | parts := strings.Split(body, "|") |
| 124 | if len(parts) != 7 { |
| 125 | t.Fatalf("body %q did not come from the tenant app at all", body) |
| 126 | } |
| 127 | seen := e2eSeen{marker: parts[0]} |
| 128 | for _, p := range parts[1:] { |
| 129 | k, v, _ := strings.Cut(p, "=") |
| 130 | switch k { |
| 131 | case "xff": |
| 132 | seen.xff = v |
| 133 | case "xfh": |
| 134 | seen.xfh = v |
| 135 | case "hop": |
| 136 | seen.hop = v |
| 137 | case "auth": |
| 138 | seen.auth = v |
| 139 | case "geo": |
| 140 | seen.geoCountry = v |
| 141 | case "hopgeo": |
| 142 | seen.hopGeo = v |
| 143 | } |
| 144 | } |
| 145 | return seen |
| 146 | } |
| 147 | |
| 148 | // ---- the deployment's certificate authority ---------------------------------- |
| 149 | |
| 150 | // e2eCA stands in for the public CA Part 2's issuer orders from. A real CA |
| 151 | // rather than a self-signed leaf per host, because every handshake in this file |
| 152 | // is verified the way a browser verifies one — RootCAs, hostname, expiry — and |
| 153 | // a leaf trusted as its own root would quietly excuse a chain that production |
| 154 | // would reject. |
| 155 | type e2eCA struct { |
| 156 | root *x509.Certificate |
| 157 | key *ecdsa.PrivateKey |
| 158 | pool *x509.CertPool |
| 159 | } |
| 160 | |
| 161 | func newE2ECA(t *testing.T) *e2eCA { |
| 162 | t.Helper() |
| 163 | key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) |
| 164 | if err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | tmpl := &x509.Certificate{ |
| 168 | SerialNumber: big.NewInt(1), |
| 169 | Subject: pkix.Name{CommonName: "CARLOS backhaul test CA"}, |
| 170 | NotBefore: time.Now().Add(-time.Hour), |
| 171 | NotAfter: time.Now().Add(365 * 24 * time.Hour), |
| 172 | KeyUsage: x509.KeyUsageCertSign, |
| 173 | BasicConstraintsValid: true, |
| 174 | IsCA: true, |
| 175 | } |
| 176 | der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) |
| 177 | if err != nil { |
| 178 | t.Fatal(err) |
| 179 | } |
| 180 | root, err := x509.ParseCertificate(der) |
| 181 | if err != nil { |
| 182 | t.Fatal(err) |
| 183 | } |
| 184 | pool := x509.NewCertPool() |
| 185 | pool.AddCert(root) |
| 186 | return &e2eCA{root: root, key: key, pool: pool} |
| 187 | } |
| 188 | |
| 189 | // publish mints a certificate for host and writes it to control/certs/<host>, |
| 190 | // in autocert's own cache-entry shape and through putCert — byte-for-byte what |
| 191 | // the issuer's publishingCache leaves in the bucket. Every edge in the |
| 192 | // deployment then serves it; none of them issued it. |
| 193 | func (ca *e2eCA) publish(t *testing.T, s release.Store, host string) *x509.Certificate { |
| 194 | t.Helper() |
| 195 | key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) |
| 196 | if err != nil { |
| 197 | t.Fatal(err) |
| 198 | } |
| 199 | serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 96)) |
| 200 | if err != nil { |
| 201 | t.Fatal(err) |
| 202 | } |
| 203 | tmpl := &x509.Certificate{ |
| 204 | SerialNumber: serial, |
| 205 | Subject: pkix.Name{CommonName: host}, |
| 206 | DNSNames: []string{host}, |
| 207 | NotBefore: time.Now().Add(-time.Hour), |
| 208 | NotAfter: time.Now().Add(90 * 24 * time.Hour), |
| 209 | KeyUsage: x509.KeyUsageDigitalSignature, |
| 210 | ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, |
| 211 | } |
| 212 | der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.root, &key.PublicKey, ca.key) |
| 213 | if err != nil { |
| 214 | t.Fatal(err) |
| 215 | } |
| 216 | leaf, err := x509.ParseCertificate(der) |
| 217 | if err != nil { |
| 218 | t.Fatal(err) |
| 219 | } |
| 220 | raw, err := encodeWildcardPEM(&tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: leaf}, key) |
| 221 | if err != nil { |
| 222 | t.Fatal(err) |
| 223 | } |
| 224 | putBucket(t, s, host, raw) |
| 225 | return leaf |
| 226 | } |
| 227 | |
| 228 | // ---- one edge ---------------------------------------------------------------- |
| 229 | |
| 230 | // backhaulEdge is one box: a real Edge over its OWN handle on the shared |
| 231 | // bucket, with the full handshake chain and the full :80 chain composed the way |
| 232 | // RunWithActivatorReady composes them, on real loopback listeners. |
| 233 | type backhaulEdge struct { |
| 234 | edge *Edge |
| 235 | store release.Store |
| 236 | homes *homeMap |
| 237 | mirror *certMirror |
| 238 | mgr *autocert.Manager |
| 239 | // local is the RAW autocert cache — this box's own disk, which starts |
| 240 | // empty and is only ever written by the mirror. |
| 241 | local autocert.Cache |
| 242 | |
| 243 | tlsAddr, plainAddr string |
| 244 | // calls counts requests that reached this box's 443 door, refusals |
| 245 | // included: "exactly one dial" and "never dialled at all" are both this |
| 246 | // number moving, or not, on the OTHER box. |
| 247 | calls atomic.Int64 |
| 248 | // plainCalls is the same for :80. |
| 249 | plainCalls atomic.Int64 |
| 250 | // peerAs, when set to an address, is what this box's 443 door reports as |
| 251 | // the TCP peer of every request (the port is kept). A sending box dials |
| 252 | // from its own public address in production; on loopback every box is |
| 253 | // 127.0.0.1, so a test that needs the peer to be that address says so |
| 254 | // here. Unset, the peer is the real one. |
| 255 | peerAs atomic.Value // string |
| 256 | |
| 257 | client *http.Client |
| 258 | } |
| 259 | |
| 260 | // newBackhaulEdge builds one box. routes are host → upstream address; anything |
| 261 | // not listed is a host this box genuinely does not serve. |
| 262 | func newBackhaulEdge(t *testing.T, bucketDir, box string, ca *e2eCA, from *net.TCPAddr, routes map[string]string) *backhaulEdge { |
| 263 | t.Helper() |
| 264 | be := &backhaulEdge{store: release.NewDirStore(bucketDir)} |
| 265 | reg := openReg(t) |
| 266 | for host, addr := range routes { |
| 267 | if err := reg.Upsert(registry.Route{Host: host, Kind: "service", Addr: addr, Status: "ready"}); err != nil { |
| 268 | t.Fatalf("upsert %s: %v", host, err) |
| 269 | } |
| 270 | } |
| 271 | e, err := New(reg, nil) |
| 272 | if err != nil { |
| 273 | t.Fatalf("New: %v", err) |
| 274 | } |
| 275 | t.Cleanup(e.probers.Stop) |
| 276 | be.edge = e |
| 277 | e.staticStore = be.store |
| 278 | WithBoxLabel(box)(e) |
| 279 | // Not the issuer, and never able to become one: the gate is shut and the |
| 280 | // manager below points at a dead directory. Everything this box serves on |
| 281 | // 443 therefore came out of the bucket. |
| 282 | WithIssuerGate(func() (context.Context, bool) { return nil, false })(e) |
| 283 | |
| 284 | // The RAW cache, kept before the publishing wrapper goes on (edge.go). |
| 285 | be.local = autocert.DirCache(t.TempDir()) |
| 286 | cache := e.armCertIssuance(be.local) |
| 287 | be.mirror = e.newCertMirror(be.local) |
| 288 | if be.mirror == nil { |
| 289 | t.Fatal("a store-backed edge must have a mirror to serve from") |
| 290 | } |
| 291 | be.mgr = offlineManager(t, cache, e.issuerHostPolicy()) |
| 292 | tlsCfg := be.mgr.TLSConfig() |
| 293 | tlsCfg.GetCertificate = be.mirror.wrapGetCertificate(tlsCfg.GetCertificate) |
| 294 | |
| 295 | be.homes = e.newHomeMap() |
| 296 | if be.homes == nil { |
| 297 | t.Fatal("a store-backed edge must have a home map") |
| 298 | } |
| 299 | e.homes.Store(be.homes) |
| 300 | |
| 301 | // 443: the edge itself, counted. |
| 302 | https := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 303 | be.calls.Add(1) |
| 304 | if p, _ := be.peerAs.Load().(string); p != "" { |
| 305 | _, port, _ := net.SplitHostPort(r.RemoteAddr) |
| 306 | r.RemoteAddr = net.JoinHostPort(p, port) |
| 307 | } |
| 308 | e.ServeHTTP(w, r) |
| 309 | })) |
| 310 | https.TLS = tlsCfg |
| 311 | https.StartTLS() |
| 312 | t.Cleanup(https.Close) |
| 313 | be.tlsAddr = https.Listener.Addr().String() |
| 314 | |
| 315 | // :80, outermost first, exactly as RunWithActivatorReady composes it: the |
| 316 | // mirror's challenge handler, then autocert's, then the redirect fallback. |
| 317 | // Composed ONCE, before anything listens, for the reason edge.go composes |
| 318 | // it there — HTTPHandler is what arms autocert's HTTP-01 path. |
| 319 | acmeAndFallback := be.mirror.challengeHandler(be.mgr.HTTPHandler(e.httpFallbackHandler())) |
| 320 | plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 321 | be.plainCalls.Add(1) |
| 322 | acmeAndFallback.ServeHTTP(w, r) |
| 323 | })) |
| 324 | t.Cleanup(plain.Close) |
| 325 | be.plainAddr = plain.Listener.Addr().String() |
| 326 | |
| 327 | be.client = e2eClientFor(be, ca, from) |
| 328 | return be |
| 329 | } |
| 330 | |
| 331 | // e2eClientFor is a client that resolves every hostname to ONE box's two |
| 332 | // listeners — the deployment's DNS, in other words, pointed wherever the test |
| 333 | // wants this request to land. https connections are handshaken here rather than |
| 334 | // by the transport so that the client verifies the edge's certificate against |
| 335 | // the test CA, by name, exactly as a browser would: a mirrored certificate that |
| 336 | // did not actually cover the host would fail here and not quietly downstream. |
| 337 | func e2eClientFor(be *backhaulEdge, ca *e2eCA, from *net.TCPAddr) *http.Client { |
| 338 | dialer := &net.Dialer{Timeout: 10 * time.Second, LocalAddr: from} |
| 339 | return &http.Client{ |
| 340 | Timeout: 30 * time.Second, |
| 341 | Transport: &http.Transport{ |
| 342 | DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { |
| 343 | return dialer.DialContext(ctx, network, be.plainAddr) |
| 344 | }, |
| 345 | DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { |
| 346 | name, _, err := net.SplitHostPort(addr) |
| 347 | if err != nil { |
| 348 | return nil, err |
| 349 | } |
| 350 | raw, err := dialer.DialContext(ctx, network, be.tlsAddr) |
| 351 | if err != nil { |
| 352 | return nil, err |
| 353 | } |
| 354 | conn := tls.Client(raw, &tls.Config{ServerName: name, RootCAs: ca.pool, MinVersion: tls.VersionTLS12}) |
| 355 | if err := conn.HandshakeContext(ctx); err != nil { |
| 356 | raw.Close() |
| 357 | return nil, err |
| 358 | } |
| 359 | return conn, nil |
| 360 | }, |
| 361 | }, |
| 362 | } |
| 363 | } |
| 364 | |
| 365 | // ---- the scene --------------------------------------------------------------- |
| 366 | |
| 367 | type backhaulScene struct { |
| 368 | ca *e2eCA |
| 369 | seed release.Store // the housekeeper's handle: what writes the control plane |
| 370 | a, b *backhaulEdge |
| 371 | key []byte // the deployment's mesh key, as both edges mirror it |
| 372 | // leaf is the certificate the bucket carries per host. |
| 373 | leaf map[string]*x509.Certificate |
| 374 | // clientIP is the source address the test client dials from, or "" if the |
| 375 | // platform would not give us a second loopback address. |
| 376 | clientIP string |
| 377 | deadEIP string |
| 378 | } |
| 379 | |
| 380 | func newBackhaulScene(t *testing.T) *backhaulScene { |
| 381 | t.Helper() |
| 382 | e2eQuietLogs(t) |
| 383 | bucket := t.TempDir() |
| 384 | s := &backhaulScene{ |
| 385 | seed: release.NewDirStore(bucket), |
| 386 | ca: newE2ECA(t), |
| 387 | leaf: map[string]*x509.Certificate{}, |
| 388 | deadEIP: e2eDeadEIP(t), |
| 389 | } |
| 390 | |
| 391 | // The housekeeper's one-time mint (Task 2), and the certs Part 2's issuer |
| 392 | // publishes. Both edges read them; neither wrote them. |
| 393 | if err := MintMeshKey(context.Background(), s.seed); err != nil { |
| 394 | t.Fatalf("mint mesh key: %v", err) |
| 395 | } |
| 396 | for _, host := range []string{e2eHomedHost, e2eMovedHost, e2eDeadHost, e2eUnhomedHost, e2eSenderHost} { |
| 397 | s.leaf[host] = s.ca.publish(t, s.seed, host) |
| 398 | } |
| 399 | |
| 400 | from := e2eClientAddr(t) |
| 401 | if from != nil { |
| 402 | s.clientIP = from.IP.String() |
| 403 | } |
| 404 | s.a = newBackhaulEdge(t, bucket, "annie", s.ca, from, map[string]string{ |
| 405 | e2eHomedHost: e2eApp(t, e2eHomeMarker).Listener.Addr().String(), |
| 406 | }) |
| 407 | s.b = newBackhaulEdge(t, bucket, "betty", s.ca, from, map[string]string{ |
| 408 | e2eSenderHost: e2eApp(t, e2eSenderMarker).Listener.Addr().String(), |
| 409 | }) |
| 410 | |
| 411 | // The dial seams, armed for the test's lifetime: 443 is an ephemeral port |
| 412 | // here, and the home edge's certificate is signed by the test CA rather |
| 413 | // than a public one. Nothing about verification is weakened — the hop still |
| 414 | // checks the name and the chain (see backhaulRootCAs' own doc). |
| 415 | // |
| 416 | // One consequence of backhaulPort being a single global: every box in this |
| 417 | // scene is dialled on the SAME port, so a box's EIP is the whole of its |
| 418 | // identity on the wire. That is why the dead box below gets an address of |
| 419 | // its own rather than a port of its own. |
| 420 | eip, port, err := net.SplitHostPort(s.a.tlsAddr) |
| 421 | if err != nil { |
| 422 | t.Fatalf("split home edge address: %v", err) |
| 423 | } |
| 424 | oldPort, oldCAs := backhaulPort, backhaulRootCAs |
| 425 | backhaulPort, backhaulRootCAs = port, s.ca.pool |
| 426 | t.Cleanup(func() { backhaulPort, backhaulRootCAs = oldPort, oldCAs }) |
| 427 | |
| 428 | // The per-box edges files, Task 1's schema, written straight into the store |
| 429 | // (the publishing loop is internal/host's and is not in this process). |
| 430 | bEIP, _, err := net.SplitHostPort(s.b.tlsAddr) |
| 431 | if err != nil { |
| 432 | t.Fatalf("split sending edge address: %v", err) |
| 433 | } |
| 434 | s.publishEdgesFile(t, "annie", eip, e2eHomedHost, e2eMovedHost) |
| 435 | s.publishEdgesFile(t, "betty", bEIP, e2eSenderHost) |
| 436 | s.publishEdgesFile(t, "carol", s.deadEIP, e2eDeadHost) |
| 437 | return s |
| 438 | } |
| 439 | |
| 440 | // publishEdgesFile writes one box's control/edges/<box>.json — homedOn's shape, |
| 441 | // written through the seed handle so neither edge is the publisher of its own |
| 442 | // map (which is the production writer arrangement: the box's own host loop |
| 443 | // writes it, and every edge only ever READS the prefix). |
| 444 | func (s *backhaulScene) publishEdgesFile(t *testing.T, box, eip string, hosts ...string) { |
| 445 | t.Helper() |
| 446 | entries := make([]string, 0, len(hosts)) |
| 447 | for _, h := range hosts { |
| 448 | entries = append(entries, fmt.Sprintf("%q:{\"role\":\"home\"}", h)) |
| 449 | } |
| 450 | putJSON(t, s.seed, "control/edges/"+box+".json", fmt.Sprintf( |
| 451 | `{"v":1,"box":%q,"eip":%q,"hosts":{%s}}`, box, eip, strings.Join(entries, ","))) |
| 452 | } |
| 453 | |
| 454 | // converge drives every mirror and every map ONE poll forward. Explicit, never |
| 455 | // waited for: the loops (certMirror.run, homeMap.run) are 30s tickers, and a |
| 456 | // test that slept for them would be testing the clock. |
| 457 | func (s *backhaulScene) converge(t *testing.T) { |
| 458 | t.Helper() |
| 459 | for _, be := range []*backhaulEdge{s.a, s.b} { |
| 460 | if err := be.mirror.Refresh(context.Background()); err != nil { |
| 461 | t.Fatalf("cert mirror refresh: %v", err) |
| 462 | } |
| 463 | refreshHomeMap(t, be.homes) |
| 464 | } |
| 465 | key, next := s.a.homes.meshKeys() |
| 466 | if len(key) == 0 || len(next) != 0 { |
| 467 | t.Fatalf("mesh keys mirrored as current=%d next=%d bytes, want one current key", len(key), len(next)) |
| 468 | } |
| 469 | s.key = key |
| 470 | } |
| 471 | |
| 472 | // get drives one client request at one box and reads the whole body. follow |
| 473 | // says whether redirects are followed — the :80 leg wants both answers. |
| 474 | func (s *backhaulScene) get(t *testing.T, be *backhaulEdge, url string, headers map[string]string, follow bool) (*http.Response, string) { |
| 475 | t.Helper() |
| 476 | resp, err := s.try(be, url, headers, follow) |
| 477 | if err != nil { |
| 478 | t.Fatalf("GET %s: %v", url, err) |
| 479 | } |
| 480 | body, err := io.ReadAll(resp.Body) |
| 481 | resp.Body.Close() |
| 482 | if err != nil { |
| 483 | t.Fatalf("read body of %s: %v", url, err) |
| 484 | } |
| 485 | return resp, string(body) |
| 486 | } |
| 487 | |
| 488 | func (s *backhaulScene) try(be *backhaulEdge, url string, headers map[string]string, follow bool) (*http.Response, error) { |
| 489 | req, err := http.NewRequest(http.MethodGet, url, nil) |
| 490 | if err != nil { |
| 491 | return nil, err |
| 492 | } |
| 493 | for k, v := range headers { |
| 494 | req.Header.Set(k, v) |
| 495 | } |
| 496 | client := be.client |
| 497 | if !follow { |
| 498 | c := *be.client |
| 499 | c.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } |
| 500 | client = &c |
| 501 | } |
| 502 | return client.Do(req) |
| 503 | } |
| 504 | |
| 505 | // e2eClientAddr is the source address the test client dials FROM: a second |
| 506 | // loopback address, so the client is distinguishable ON THE WIRE from the |
| 507 | // sending edge, which dials the home edge from the default one. That is what |
| 508 | // turns "X-Forwarded-For names the client, not the edge that forwarded it" into |
| 509 | // an assertion rather than a coincidence of both being 127.0.0.1. |
| 510 | // |
| 511 | // nil on a platform that will not hand out a second loopback address; the |
| 512 | // chain-LENGTH assertions still hold there, and the identity ones are skipped |
| 513 | // explicitly rather than silently passing. |
| 514 | func e2eClientAddr(t *testing.T) *net.TCPAddr { |
| 515 | t.Helper() |
| 516 | ln, err := net.Listen("tcp", "127.0.0.2:0") |
| 517 | if err != nil { |
| 518 | return nil |
| 519 | } |
| 520 | ln.Close() |
| 521 | return &net.TCPAddr{IP: net.ParseIP("127.0.0.2")} |
| 522 | } |
| 523 | |
| 524 | // e2eDeadEIP is an address nothing listens on, for the poisoned-map case: a |
| 525 | // loopback one when the platform allows it, so the dial is REFUSED immediately |
| 526 | // rather than hanging. The documentation address is the fallback, and costs the |
| 527 | // backhaul dial timeout instead — the same 502, more slowly. |
| 528 | func e2eDeadEIP(t *testing.T) string { |
| 529 | t.Helper() |
| 530 | if ln, err := net.Listen("tcp", "127.0.0.9:0"); err == nil { |
| 531 | ln.Close() |
| 532 | return "127.0.0.9" |
| 533 | } |
| 534 | return "192.0.2.1" // TEST-NET-1, RFC 5737 |
| 535 | } |
| 536 | |
| 537 | // ---- the scenario ------------------------------------------------------------ |
| 538 | |
| 539 | func TestBackhaulE2EAcrossTwoEdges(t *testing.T) { |
| 540 | s := newBackhaulScene(t) |
| 541 | |
| 542 | // Before either mirror has polled, edge B holds nothing for the homed host: |
| 543 | // empty mirror, empty local cache, autocert gated shut. The handshake must |
| 544 | // FAIL — otherwise every success below could be autocert's own doing, or a |
| 545 | // certificate that was on B's disk all along. |
| 546 | if _, err := s.try(s.b, "https://"+e2eHomedHost+"/", nil, true); err == nil { |
| 547 | t.Fatal("the sending edge completed a handshake for a certificate no box has published to it yet") |
| 548 | } |
| 549 | s.converge(t) |
| 550 | |
| 551 | t.Run("P1 the sending edge answers a hello for a host it does not route", func(t *testing.T) { |
| 552 | // The premise, asserted rather than assumed: B has no route for this |
| 553 | // host, will not issue for it, and its own manager refuses it outright. |
| 554 | if _, ok := s.b.edge.lookupUpstream(e2eHomedHost); ok { |
| 555 | t.Fatal("test bug: the sending edge must not route the host it is supposed to backhaul") |
| 556 | } |
| 557 | if s.b.edge.Allowed(e2eHomedHost) { |
| 558 | t.Fatal("test bug: the sending edge must not claim the host for issuance either") |
| 559 | } |
| 560 | if _, err := s.b.mgr.GetCertificate(ecdsaHello(e2eHomedHost)); err == nil { |
| 561 | t.Fatal("the sending edge's own autocert manager must refuse a host it does not serve") |
| 562 | } |
| 563 | |
| 564 | // And yet a real client, verifying against the CA by name, completes the |
| 565 | // handshake — with the certificate the BUCKET carries. This is the |
| 566 | // invariant the feature rides on: without it a remote-homed host dies at |
| 567 | // the TLS layer on every edge but its home, and nothing downstream ever |
| 568 | // runs. |
| 569 | resp, _ := s.get(t, s.b, "https://"+e2eHomedHost+"/", nil, true) |
| 570 | if resp.TLS == nil || len(resp.TLS.PeerCertificates) == 0 { |
| 571 | t.Fatal("no peer certificate — the handshake did not happen") |
| 572 | } |
| 573 | if got := resp.TLS.PeerCertificates[0]; !got.Equal(s.leaf[e2eHomedHost]) { |
| 574 | t.Fatalf("the sending edge served a certificate that is not the mirrored one (subject %q)", got.Subject.CommonName) |
| 575 | } |
| 576 | if c := s.b.mirror.certificate(e2eHomedHost, false); c == nil { |
| 577 | t.Fatal("the mirror map holds no certificate for the host — it cannot have been the source") |
| 578 | } |
| 579 | }) |
| 580 | |
| 581 | t.Run("the marker body backhauls, the chain names the client, one dial", func(t *testing.T) { |
| 582 | home, sender := s.a.calls.Load(), s.b.calls.Load() |
| 583 | resp, body := s.get(t, s.b, "https://"+e2eHomedHost+"/deep?q=1", nil, true) |
| 584 | if resp.StatusCode != http.StatusOK { |
| 585 | t.Fatalf("status = %d, want 200 — a remote-homed host is served through its home edge", resp.StatusCode) |
| 586 | } |
| 587 | seen := e2eParseApp(t, body) |
| 588 | if seen.marker != e2eHomeMarker { |
| 589 | t.Fatalf("marker = %q, want the HOME edge's app (%q)", seen.marker, e2eHomeMarker) |
| 590 | } |
| 591 | if got := s.a.calls.Load() - home; got != 1 { |
| 592 | t.Fatalf("the home edge saw %d request(s), want exactly one — one hop, ever", got) |
| 593 | } |
| 594 | if got := s.b.calls.Load() - sender; got != 1 { |
| 595 | t.Fatalf("the sending edge saw %d request(s) for one client request", got) |
| 596 | } |
| 597 | // The chain the app sees is a single element and it is the CLIENT's |
| 598 | // address — not the sending edge's, which is a different loopback |
| 599 | // address here precisely so the two can be told apart. Every consumer in |
| 600 | // this deployment takes the last element as the caller |
| 601 | // (internal/console/authcli.go's clientIP), so an appended edge would |
| 602 | // bill, rate-limit and log a real visitor as infrastructure. |
| 603 | if strings.Contains(seen.xff, "+") || strings.Contains(seen.xff, ",") { |
| 604 | t.Fatalf("X-Forwarded-For = %q, want a single-element chain", seen.xff) |
| 605 | } |
| 606 | if s.clientIP != "" && seen.xff != s.clientIP { |
| 607 | t.Fatalf("X-Forwarded-For = %q, want the client's own address %q", seen.xff, s.clientIP) |
| 608 | } |
| 609 | if seen.xfh != e2eHomedHost { |
| 610 | t.Fatalf("X-Forwarded-Host = %q, want the host the client asked for", seen.xfh) |
| 611 | } |
| 612 | // The hop vocabulary is the edges' own and stops at the home edge. |
| 613 | if seen.hop != "" || seen.auth != "" { |
| 614 | t.Fatalf("hop=%q auth=%q reached the tenant's app", seen.hop, seen.auth) |
| 615 | } |
| 616 | }) |
| 617 | |
| 618 | t.Run("the sending edge still serves its own routes without dialling", func(t *testing.T) { |
| 619 | home := s.a.calls.Load() |
| 620 | resp, body := s.get(t, s.b, "https://"+e2eSenderHost+"/", nil, true) |
| 621 | if resp.StatusCode != http.StatusOK { |
| 622 | t.Fatalf("status = %d, want 200", resp.StatusCode) |
| 623 | } |
| 624 | if seen := e2eParseApp(t, body); seen.marker != e2eSenderMarker { |
| 625 | t.Fatalf("marker = %q, want the sending edge's OWN app", seen.marker) |
| 626 | } |
| 627 | if got := s.a.calls.Load() - home; got != 0 { |
| 628 | t.Fatalf("the home edge saw %d request(s) for a host the sending edge serves itself", got) |
| 629 | } |
| 630 | }) |
| 631 | |
| 632 | t.Run("the :80 leg redirects, and the redirect it hands out backhauls", func(t *testing.T) { |
| 633 | // A first-time visitor typing a bare hostname lands on whichever edge |
| 634 | // DNS picked, on :80. Task 5's fix: a remote-homed host is routable, so |
| 635 | // it gets the ordinary redirect rather than a 404 for an app that is up. |
| 636 | plain := s.b.plainCalls.Load() |
| 637 | resp, _ := s.get(t, s.b, "http://"+e2eHomedHost+"/deep?q=1", nil, false) |
| 638 | if resp.StatusCode != http.StatusFound { |
| 639 | t.Fatalf("status = %d, want 302 — :80 must send a remote-homed host to 443", resp.StatusCode) |
| 640 | } |
| 641 | if got := s.b.plainCalls.Load() - plain; got != 1 { |
| 642 | t.Fatalf("the sending edge's :80 door saw %d request(s), want one — this leg must be plain HTTP", got) |
| 643 | } |
| 644 | if got, want := resp.Header.Get("Location"), "https://"+e2eHomedHost+"/deep?q=1"; got != want { |
| 645 | t.Fatalf("Location = %q, want %q", got, want) |
| 646 | } |
| 647 | // A host nothing in the deployment homes keeps the unchanged 404 on the |
| 648 | // same handler, so the redirect is a routing answer and not a blanket one. |
| 649 | if got, _ := s.get(t, s.b, "http://"+e2eUnhomedHost+"/", nil, false); got.StatusCode != http.StatusNotFound { |
| 650 | t.Fatalf("status = %d, want the unchanged 404 for a host nothing homes", got.StatusCode) |
| 651 | } |
| 652 | |
| 653 | // Both legs, one client call: the visitor follows the redirect and ends |
| 654 | // on the home edge's app, having crossed :80 here and 443 there. |
| 655 | home := s.a.calls.Load() |
| 656 | final, body := s.get(t, s.b, "http://"+e2eHomedHost+"/deep?q=1", nil, true) |
| 657 | if final.StatusCode != http.StatusOK { |
| 658 | t.Fatalf("status = %d after following the redirect, want 200", final.StatusCode) |
| 659 | } |
| 660 | if final.Request.URL.Scheme != "https" { |
| 661 | t.Fatalf("the followed request ended on %q, want https", final.Request.URL.Scheme) |
| 662 | } |
| 663 | if final.TLS == nil { |
| 664 | t.Fatal("the second leg was not TLS") |
| 665 | } |
| 666 | if seen := e2eParseApp(t, body); seen.marker != e2eHomeMarker { |
| 667 | t.Fatalf("marker = %q, want the home edge's app at the end of both legs", seen.marker) |
| 668 | } |
| 669 | if got := s.a.calls.Load() - home; got != 1 { |
| 670 | t.Fatalf("the home edge saw %d request(s) across the two legs, want one", got) |
| 671 | } |
| 672 | }) |
| 673 | |
| 674 | t.Run("a poisoned map is a bad gateway, and never a loop", func(t *testing.T) { |
| 675 | if _, box, ok := s.b.homes.Home(e2eDeadHost); !ok || box != "carol" { |
| 676 | t.Fatalf("test bug: the map must name the dead box for %s (got %q, %v)", e2eDeadHost, box, ok) |
| 677 | } |
| 678 | home, sender := s.a.calls.Load(), s.b.calls.Load() |
| 679 | resp, body := s.get(t, s.b, "https://"+e2eDeadHost+"/", nil, true) |
| 680 | if resp.StatusCode != http.StatusBadGateway { |
| 681 | t.Fatalf("status = %d, want 502 — this edge believed a specific box was there", resp.StatusCode) |
| 682 | } |
| 683 | if got := resp.Header.Get("Retry-After"); got != "" { |
| 684 | t.Fatalf("Retry-After = %q — that answer is reserved for KNOWING nobody is home", got) |
| 685 | } |
| 686 | if strings.Contains(body, e2eHomeMarker) { |
| 687 | t.Fatal("a dead home somehow served the app") |
| 688 | } |
| 689 | if got := s.b.calls.Load() - sender; got != 1 { |
| 690 | t.Fatalf("the sending edge handled %d request(s) for one client request — a failed dial must not be retried around the fleet", got) |
| 691 | } |
| 692 | if got := s.a.calls.Load() - home; got != 0 { |
| 693 | t.Fatalf("the live home edge saw %d request(s) for a host it does not home — a failed dial must not scan for a second target", got) |
| 694 | } |
| 695 | }) |
| 696 | |
| 697 | t.Run("a stale map gets the home edge's 503, and the sender does not re-dial", func(t *testing.T) { |
| 698 | // The map still homes this host on A; A no longer routes it. That is |
| 699 | // what one publish cycle of a placement move looks like from here. |
| 700 | if _, box, ok := s.b.homes.Home(e2eMovedHost); !ok || box != "annie" { |
| 701 | t.Fatalf("test bug: the map must still name the home edge for %s (got %q, %v)", e2eMovedHost, box, ok) |
| 702 | } |
| 703 | if _, ok := s.a.edge.lookupUpstream(e2eMovedHost); ok { |
| 704 | t.Fatal("test bug: the home edge must NOT serve the moved host") |
| 705 | } |
| 706 | home, sender := s.a.calls.Load(), s.b.calls.Load() |
| 707 | resp, body := s.get(t, s.b, "https://"+e2eMovedHost+"/", nil, true) |
| 708 | if resp.StatusCode != http.StatusServiceUnavailable { |
| 709 | t.Fatalf("status = %d, want the home edge's 503", resp.StatusCode) |
| 710 | } |
| 711 | if got := resp.Header.Get("Retry-After"); got != "2" { |
| 712 | t.Fatalf("Retry-After = %q, want 2 — converge heals this within a publish cycle", got) |
| 713 | } |
| 714 | // The refusal reaches the client as the home edge wrote it, and says |
| 715 | // nothing about where the host went. |
| 716 | if strings.Contains(body, e2eMovedHost) || strings.Contains(body, "annie") || strings.Contains(body, "betty") { |
| 717 | t.Fatalf("body = %q — the refusal must not name the host or any box", body) |
| 718 | } |
| 719 | if got := s.a.calls.Load() - home; got != 1 { |
| 720 | t.Fatalf("the home edge saw %d request(s), want exactly one — a 503 is an answer, not a reason to try again", got) |
| 721 | } |
| 722 | if got := s.b.calls.Load() - sender; got != 1 { |
| 723 | t.Fatalf("the sending edge handled %d request(s) for one client request", got) |
| 724 | } |
| 725 | }) |
| 726 | |
| 727 | t.Run("a tampered hop reaches the home edge as an ordinary client", func(t *testing.T) { |
| 728 | // Tampering happens on the wire INTO the home edge: a genuine signature |
| 729 | // this deployment's own key produced, with one hex nibble flipped, plus |
| 730 | // the forwarding claims a tamperer would want believed. |
| 731 | tampered := []byte(signHop(s.key, e2eHomedHost, time.Now())) |
| 732 | tampered[len(tampered)-1] ^= 'f' ^ '0' |
| 733 | headers := map[string]string{ |
| 734 | hopHeader: "1", |
| 735 | hopAuthHeader: string(tampered), |
| 736 | "X-Forwarded-For": "203.0.113.9", |
| 737 | "X-Forwarded-Host": "evil.test", |
| 738 | "X-Forwarded-Proto": "https", |
| 739 | } |
| 740 | |
| 741 | resp, body := s.get(t, s.a, "https://"+e2eHomedHost+"/", headers, true) |
| 742 | if resp.StatusCode != http.StatusOK { |
| 743 | t.Fatalf("status = %d, want 200 — a failed verification is 'you are an ordinary client', not a rejection", resp.StatusCode) |
| 744 | } |
| 745 | seen := e2eParseApp(t, body) |
| 746 | if seen.marker != e2eHomeMarker { |
| 747 | t.Fatalf("marker = %q, want the home edge's app", seen.marker) |
| 748 | } |
| 749 | if strings.Contains(seen.xff, "203.0.113.9") { |
| 750 | t.Fatalf("X-Forwarded-For = %q — a chain nothing vouched for must be discarded", seen.xff) |
| 751 | } |
| 752 | if s.clientIP != "" && seen.xff != s.clientIP { |
| 753 | t.Fatalf("X-Forwarded-For = %q, want the tamperer's own address %q", seen.xff, s.clientIP) |
| 754 | } |
| 755 | if seen.xfh == "evil.test" { |
| 756 | t.Fatalf("X-Forwarded-Host = %q — an unverified sender does not get to state it", seen.xfh) |
| 757 | } |
| 758 | if seen.hop != "" || seen.auth != "" { |
| 759 | t.Fatalf("hop=%q auth=%q reached the app — hop vocabulary is stripped either way", seen.hop, seen.auth) |
| 760 | } |
| 761 | |
| 762 | // And no hop SEMANTICS at all: for a host the home edge does not serve, |
| 763 | // a tampered hop gets the ordinary unknown-host answer. The same request |
| 764 | // with an intact signature gets the stale-map refusal instead — the |
| 765 | // signature is the whole of the difference, which is what makes the 503 |
| 766 | // unreachable to a stranger who cannot mint one. |
| 767 | bad := map[string]string{hopHeader: "1", hopAuthHeader: string(tampered)} |
| 768 | if got, _ := s.get(t, s.a, "https://"+e2eUnhomedHost+"/", bad, true); got.StatusCode != http.StatusNotFound { |
| 769 | t.Fatalf("status = %d, want the unchanged 404 — a tampered hop earns no hop semantics", got.StatusCode) |
| 770 | } else if ra := got.Header.Get("Retry-After"); ra != "" { |
| 771 | t.Fatalf("Retry-After = %q — a stranger must not be told to come back", ra) |
| 772 | } |
| 773 | good := map[string]string{hopHeader: "1", hopAuthHeader: signHop(s.key, e2eUnhomedHost, time.Now())} |
| 774 | if got, _ := s.get(t, s.a, "https://"+e2eUnhomedHost+"/", good, true); got.StatusCode != http.StatusServiceUnavailable { |
| 775 | t.Fatalf("status = %d, want 503 — an INTACT signature is what buys the stale-map refusal", got.StatusCode) |
| 776 | } |
| 777 | }) |
| 778 | } |
| 779 | |
| 780 | // ---- attribution: the hop annotation, end to end ----------------------------- |
| 781 | |
| 782 | // The attribution half of the egress ruling, on the real two-edge scene: a |
| 783 | // backhauled request is metered ONCE, on the box that holds the route's |
| 784 | // authoritative row, and that one row also carries the hop annotation. The |
| 785 | // sending edge records nothing at all for the host — it has no route row and |
| 786 | // therefore no account or app to bill, which is exactly why the annotation |
| 787 | // lives here and not there. |
| 788 | func TestHopServedRequestMetersOnceWithHopCounters(t *testing.T) { |
| 789 | s := newBackhaulScene(t) |
| 790 | s.converge(t) |
| 791 | |
| 792 | home, sender := NewUsageMeter(), NewUsageMeter() |
| 793 | s.a.edge.EnableUsage(home) |
| 794 | s.b.edge.EnableUsage(sender) |
| 795 | |
| 796 | resp, body := s.get(t, s.b, "https://"+e2eHomedHost+"/deep?q=1", nil, true) |
| 797 | if resp.StatusCode != http.StatusOK { |
| 798 | t.Fatalf("status = %d, want 200", resp.StatusCode) |
| 799 | } |
| 800 | if seen := e2eParseApp(t, body); seen.marker != e2eHomeMarker { |
| 801 | t.Fatalf("marker = %q, want the home edge's app", seen.marker) |
| 802 | } |
| 803 | |
| 804 | rows := home.Drain() |
| 805 | row, ok := rows[e2eHomedHost] |
| 806 | if !ok { |
| 807 | t.Fatalf("the home edge metered nothing for %s: %+v", e2eHomedHost, rows) |
| 808 | } |
| 809 | if row.Requests != 1 { |
| 810 | t.Fatalf("row = %+v, want exactly one request — a hop is ONE metered exchange, not two", row) |
| 811 | } |
| 812 | if row.BytesOut == 0 { |
| 813 | t.Fatalf("row = %+v, want the wire bytes the home edge sent", row) |
| 814 | } |
| 815 | if row.HopRequests != 1 || row.HopBytesOut != row.BytesOut { |
| 816 | t.Fatalf("row = %+v, want the whole exchange annotated as hopped (hop_bytes_out == bytes_out)", row) |
| 817 | } |
| 818 | if row.HopBytesOut > row.BytesOut || row.HopRequests > row.Requests { |
| 819 | t.Fatalf("row = %+v — the hop counters are a subset of the totals", row) |
| 820 | } |
| 821 | |
| 822 | // The sending edge is meterless for this host BY DESIGN: no route row, no |
| 823 | // identity, so anything it wrote would be unattributed — and a second |
| 824 | // Record for one exchange is the double count this arrangement refuses. |
| 825 | if got, ok := sender.Drain()[e2eHomedHost]; ok { |
| 826 | t.Fatalf("the sending edge metered %+v for a host it has no route for", got) |
| 827 | } |
| 828 | } |
| 829 | |
| 830 | // ---- the single-box regression pin ------------------------------------------- |
| 831 | |
| 832 | // Everything above is dormant on a deployment with one platform box, and this |
| 833 | // is the pin that says so: an edge whose control/edges/ prefix holds only its |
| 834 | // OWN file serves its own routes exactly as it always has, answers the |
| 835 | // unchanged 404 for everything else, and never opens a connection to anything. |
| 836 | // |
| 837 | // The witness is a zero-call dialer — a listener that counts every connection |
| 838 | // made to the backhaul port — with the single box's own edges file naming the |
| 839 | // listener's ADDRESS as its EIP. So the one failure this pin exists to catch, |
| 840 | // an edge that offers itself as a backhaul target, lands ON the counter rather |
| 841 | // than merely failing somewhere else. |
| 842 | // |
| 843 | // The file lists a host the box does not route, which is what makes the counter |
| 844 | // a real witness rather than a formality: a route removed between two publishes |
| 845 | // leaves exactly that state for one cycle, it is the ONLY shape in which a |
| 846 | // single box could dial at all (a host it serves is served locally, before the |
| 847 | // map is ever consulted), and dropping own-box exclusion turns it into a dial to |
| 848 | // this box's own front door. |
| 849 | func TestBackhaulE2ESingleBoxNeverDials(t *testing.T) { |
| 850 | e2eQuietLogs(t) |
| 851 | bucket := t.TempDir() |
| 852 | seed := release.NewDirStore(bucket) |
| 853 | if err := MintMeshKey(context.Background(), seed); err != nil { |
| 854 | t.Fatalf("mint mesh key: %v", err) |
| 855 | } |
| 856 | |
| 857 | const soloHost = "solo.backhaul.test" |
| 858 | // A host this box's own file still claims but its registry no longer |
| 859 | // serves — one publish cycle after a route was removed. |
| 860 | const soloStaleHost = "stale.backhaul.test" |
| 861 | app := e2eApp(t, e2eHomeMarker) |
| 862 | reg := openReg(t) |
| 863 | if err := reg.Upsert(registry.Route{Host: soloHost, Kind: "service", Addr: app.Listener.Addr().String(), Status: "ready"}); err != nil { |
| 864 | t.Fatalf("upsert: %v", err) |
| 865 | } |
| 866 | e, err := New(reg, nil) |
| 867 | if err != nil { |
| 868 | t.Fatalf("New: %v", err) |
| 869 | } |
| 870 | t.Cleanup(e.probers.Stop) |
| 871 | e.staticStore = release.NewDirStore(bucket) |
| 872 | WithBoxLabel("solo")(e) |
| 873 | |
| 874 | eip, dials := e2eDialCounter(t) |
| 875 | putJSON(t, seed, "control/edges/solo.json", fmt.Sprintf( |
| 876 | `{"v":1,"box":"solo","eip":%q,"hosts":{%q:{"role":"home"},%q:{"role":"home"}}}`, |
| 877 | eip, soloHost, soloStaleHost)) |
| 878 | |
| 879 | m := e.newHomeMap() |
| 880 | refreshHomeMap(t, m) |
| 881 | e.homes.Store(m) |
| 882 | for _, host := range []string{soloHost, soloStaleHost} { |
| 883 | if _, _, ok := m.Home(host); ok { |
| 884 | t.Fatalf("%s: the only box in a deployment must never be offered as a backhaul target for its own host", host) |
| 885 | } |
| 886 | } |
| 887 | |
| 888 | if resp := hopRequest(t, e, soloHost, nil); resp.StatusCode != http.StatusOK { |
| 889 | t.Fatalf("status = %d, want 200 — a single box serves its own routes as before", resp.StatusCode) |
| 890 | } else if seen := e2eParseApp(t, readBody(t, resp)); seen.marker != e2eHomeMarker { |
| 891 | t.Fatalf("marker = %q, want the box's own app", seen.marker) |
| 892 | } |
| 893 | for _, host := range []string{soloStaleHost, "nothing.backhaul.test"} { |
| 894 | if resp := hopRequest(t, e, host, nil); resp.StatusCode != http.StatusNotFound { |
| 895 | t.Fatalf("%s: status = %d, want the unchanged 404", host, resp.StatusCode) |
| 896 | } |
| 897 | } |
| 898 | if n := dials.Load(); n != 0 { |
| 899 | t.Fatalf("%d backhaul dial(s) from a single-box deployment, want none", n) |
| 900 | } |
| 901 | n := 0 |
| 902 | e.backhaulTransports.Range(func(_, _ any) bool { n++; return true }) |
| 903 | if n != 0 { |
| 904 | t.Fatalf("%d backhaul transport(s) built, want none — nothing was ever a target", n) |
| 905 | } |
| 906 | } |
| 907 | |
| 908 | // e2eDialCounter is the zero-call witness: a listener on the backhaul port that |
| 909 | // counts connections and answers nothing. It returns the address a dial would |
| 910 | // have to name to reach it, and arms backhaulPort for the test's lifetime. |
| 911 | func e2eDialCounter(t *testing.T) (string, *atomic.Int64) { |
| 912 | t.Helper() |
| 913 | ln, err := net.Listen("tcp", "127.0.0.1:0") |
| 914 | if err != nil { |
| 915 | t.Fatalf("listen: %v", err) |
| 916 | } |
| 917 | t.Cleanup(func() { ln.Close() }) |
| 918 | var n atomic.Int64 |
| 919 | go func() { |
| 920 | for { |
| 921 | conn, err := ln.Accept() |
| 922 | if err != nil { |
| 923 | return |
| 924 | } |
| 925 | n.Add(1) |
| 926 | conn.Close() |
| 927 | } |
| 928 | }() |
| 929 | host, port, err := net.SplitHostPort(ln.Addr().String()) |
| 930 | if err != nil { |
| 931 | t.Fatalf("split: %v", err) |
| 932 | } |
| 933 | old := backhaulPort |
| 934 | backhaulPort = port |
| 935 | t.Cleanup(func() { backhaulPort = old }) |
| 936 | return host, &n |
| 937 | } |
| 938 | |
amadan