| 1 | package tunnel |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/rand" |
| 6 | "encoding/base64" |
| 7 | "encoding/json" |
| 8 | "fmt" |
| 9 | "log" |
| 10 | "math/big" |
| 11 | "net" |
| 12 | "net/http" |
| 13 | "net/http/httputil" |
| 14 | "strings" |
| 15 | "sync" |
| 16 | "sync/atomic" |
| 17 | "time" |
| 18 | |
| 19 | "github.com/coder/websocket" |
| 20 | ) |
| 21 | |
| 22 | // Gateway terminates the public side of every tunnel. It serves two kinds of |
| 23 | // host: its own control host (register/conn endpoints) and the try-* slug |
| 24 | // hosts, which it reverse-proxies down the matching tunnel. Route |
| 25 | // registration is delegated through OnRegister/OnClose so this package never |
| 26 | // touches the router's registry directly. |
| 27 | type Gateway struct { |
| 28 | ControlHost string // e.g. "tunnel.11ch.at" — where clients register |
| 29 | Domain string // e.g. "11ch.at" — slugs become try-<slug>.<domain> |
| 30 | |
| 31 | // OnRegister makes the public hostname routable (a registry row pointing |
| 32 | // back at this gateway's listener); OnClose removes it. Both must be safe |
| 33 | // to call from multiple goroutines. |
| 34 | OnRegister func(host string) error |
| 35 | OnClose func(host string) |
| 36 | |
| 37 | // Relay mode: stable keypair-claimed hostnames (docs/relay-mode.md). |
| 38 | // Off unless RelayDomain and ClaimsDB are both set — the gateway then |
| 39 | // refuses "relay" hellos, and trials behave exactly as before. |
| 40 | RelayDomain string // e.g. "example-relay.net" — relay hosts are r-<slug>.<RelayDomain> |
| 41 | ClaimsDB *Claims // claim persistence (gateway-owned sqlite) |
| 42 | MaxRelays int // claimed hostnames (0 = default 200) |
| 43 | RelayGrace time.Duration // route lifetime after a relay disconnects (0 = default 15m) |
| 44 | ClaimIdle time.Duration // free claims idle this long (0 = default 45 days) |
| 45 | |
| 46 | MaxTunnels int // concurrent tunnels (0 = default 50) |
| 47 | MaxConns int // concurrent data connections per tunnel (0 = default 32) |
| 48 | MaxPerIP int // new tunnels per IP per hour (0 = default 5) |
| 49 | MaxAge time.Duration // tunnel lifetime (0 = default 24h) |
| 50 | DialTimeout time.Duration // client dial-back wait (0 = default 15s) |
| 51 | |
| 52 | mu sync.Mutex |
| 53 | tunnels map[string]*liveTunnel // by public host |
| 54 | pending map[string]chan net.Conn |
| 55 | byIP map[string][]time.Time // register timestamps, pruned hourly |
| 56 | } |
| 57 | |
| 58 | type liveTunnel struct { |
| 59 | host string |
| 60 | control *websocket.Conn |
| 61 | proxy *httputil.ReverseProxy |
| 62 | created time.Time |
| 63 | closed chan struct{} |
| 64 | once sync.Once |
| 65 | // relay tunnels survive a disconnect: sleeping keeps the hostname |
| 66 | // routable (answering 503) for RelayGrace so a home connection blip |
| 67 | // isn't an outage; reattach cancels the grace timer. |
| 68 | relay bool |
| 69 | pubkey string |
| 70 | sleeping bool |
| 71 | grace *time.Timer |
| 72 | // conns counts live + in-flight data connections, bounding how much of the |
| 73 | // trial machine's uplink one hostile visitor can claim. |
| 74 | conns atomic.Int32 |
| 75 | } |
| 76 | |
| 77 | func (g *Gateway) maxTunnels() int { return orInt(g.MaxTunnels, 50) } |
| 78 | func (g *Gateway) maxConns() int { return orInt(g.MaxConns, 32) } |
| 79 | func (g *Gateway) maxPerIP() int { return orInt(g.MaxPerIP, 5) } |
| 80 | func (g *Gateway) maxAge() time.Duration { |
| 81 | if g.MaxAge > 0 { |
| 82 | return g.MaxAge |
| 83 | } |
| 84 | return 24 * time.Hour |
| 85 | } |
| 86 | func (g *Gateway) dialTimeout() time.Duration { |
| 87 | if g.DialTimeout > 0 { |
| 88 | return g.DialTimeout |
| 89 | } |
| 90 | return 15 * time.Second |
| 91 | } |
| 92 | func orInt(v, def int) int { |
| 93 | if v > 0 { |
| 94 | return v |
| 95 | } |
| 96 | return def |
| 97 | } |
| 98 | |
| 99 | func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| 100 | host := r.Host |
| 101 | if h, _, err := net.SplitHostPort(host); err == nil { |
| 102 | host = h |
| 103 | } |
| 104 | if host == g.ControlHost { |
| 105 | switch { |
| 106 | case r.URL.Path == "/api/tunnel/register": |
| 107 | g.handleRegister(w, r) |
| 108 | case strings.HasPrefix(r.URL.Path, "/api/tunnel/conn/"): |
| 109 | g.handleConn(w, r) |
| 110 | case r.URL.Path == "/healthz": |
| 111 | fmt.Fprintln(w, "ok") |
| 112 | default: |
| 113 | // A human wandered in; point them somewhere useful. |
| 114 | http.Redirect(w, r, "https://elevenmessenger.com/self-host/", http.StatusFound) |
| 115 | } |
| 116 | return |
| 117 | } |
| 118 | g.mu.Lock() |
| 119 | t := g.tunnels[host] |
| 120 | g.mu.Unlock() |
| 121 | if t == nil { |
| 122 | http.Error(w, "this trial space isn't running — its lchat try command has stopped", http.StatusBadGateway) |
| 123 | return |
| 124 | } |
| 125 | g.mu.Lock() |
| 126 | asleep := t.sleeping |
| 127 | g.mu.Unlock() |
| 128 | if asleep { |
| 129 | http.Error(w, "this space's machine is offline right now — it reconnects automatically when it's back", http.StatusServiceUnavailable) |
| 130 | return |
| 131 | } |
| 132 | t.proxy.ServeHTTP(w, r) |
| 133 | } |
| 134 | |
| 135 | // handleRegister accepts a client's control connection, assigns it a slug, |
| 136 | // makes the host routable, and then sits in a read loop until the client goes |
| 137 | // away (the read also services the client's keepalive pings). |
| 138 | func (g *Gateway) handleRegister(w http.ResponseWriter, r *http.Request) { |
| 139 | ip := clientIP(r) |
| 140 | if err := g.admit(ip); err != nil { |
| 141 | http.Error(w, err.Error(), http.StatusTooManyRequests) |
| 142 | return |
| 143 | } |
| 144 | c, err := websocket.Accept(w, r, nil) |
| 145 | if err != nil { |
| 146 | return |
| 147 | } |
| 148 | defer c.CloseNow() |
| 149 | c.SetReadLimit(1 << 16) |
| 150 | |
| 151 | ctx := r.Context() |
| 152 | var hello ControlMsg |
| 153 | if err := readJSON(ctx, c, &hello); err != nil || hello.Type != "hello" { |
| 154 | writeJSON(ctx, c, ControlMsg{Type: "error", Error: "expected hello"}) |
| 155 | return |
| 156 | } |
| 157 | |
| 158 | if hello.Mode == "relay" { |
| 159 | g.handleRelayRegister(ctx, c, ip) |
| 160 | return |
| 161 | } |
| 162 | |
| 163 | host, err := g.newHost() |
| 164 | if err != nil { |
| 165 | writeJSON(ctx, c, ControlMsg{Type: "error", Error: err.Error()}) |
| 166 | return |
| 167 | } |
| 168 | t := &liveTunnel{host: host, control: c, created: time.Now(), closed: make(chan struct{})} |
| 169 | t.proxy = g.proxyFor(t) |
| 170 | |
| 171 | g.mu.Lock() |
| 172 | g.tunnels[host] = t |
| 173 | g.byIP[ip] = append(g.byIP[ip], time.Now()) |
| 174 | g.mu.Unlock() |
| 175 | |
| 176 | // The hostname goes public only after the backend proves it's an Eleven |
| 177 | // instance (the client serves dial-backs from the moment it says hello). |
| 178 | if err := g.probeBackend(ctx, t); err != nil { |
| 179 | msg := "only Eleven instances can be tunnelled: " + err.Error() |
| 180 | writeJSON(ctx, c, ControlMsg{Type: "error", Error: msg}) |
| 181 | // Graceful close so the error frame is delivered before the socket |
| 182 | // dies (CloseNow would race it and the client would just see EOF). |
| 183 | c.Close(websocket.StatusPolicyViolation, "refused") |
| 184 | g.drop(t) |
| 185 | log.Printf("tunnel: %s refused (probe): %v", host, err) |
| 186 | return |
| 187 | } |
| 188 | |
| 189 | if g.OnRegister != nil { |
| 190 | if err := g.OnRegister(host); err != nil { |
| 191 | g.drop(t) |
| 192 | writeJSON(ctx, c, ControlMsg{Type: "error", Error: "couldn't publish the address, try again"}) |
| 193 | log.Printf("tunnel: register %s: %v", host, err) |
| 194 | return |
| 195 | } |
| 196 | } |
| 197 | log.Printf("tunnel: %s open (from %s)", host, ip) |
| 198 | if err := writeJSON(ctx, c, ControlMsg{Type: "ready", Host: host}); err != nil { |
| 199 | g.drop(t) |
| 200 | return |
| 201 | } |
| 202 | |
| 203 | // Hold the connection until the client leaves or the tunnel ages out. The |
| 204 | // read loop keeps ping/pong serviced; clients never send data messages. |
| 205 | ageOut := time.NewTimer(g.maxAge()) |
| 206 | defer ageOut.Stop() |
| 207 | // Keep re-checking that the backend is still an Eleven instance; two |
| 208 | // consecutive failures drop the tunnel (a client can't swap in a |
| 209 | // different local server after passing the door check). |
| 210 | reprobe := time.NewTicker(10 * time.Minute) |
| 211 | defer reprobe.Stop() |
| 212 | probeFails := 0 |
| 213 | readErr := make(chan error, 1) |
| 214 | go func() { |
| 215 | for { |
| 216 | if _, _, err := c.Read(ctx); err != nil { |
| 217 | readErr <- err |
| 218 | return |
| 219 | } |
| 220 | } |
| 221 | }() |
| 222 | for done := false; !done; { |
| 223 | select { |
| 224 | case <-readErr: |
| 225 | done = true |
| 226 | case <-reprobe.C: |
| 227 | if err := g.probeBackend(ctx, t); err != nil { |
| 228 | probeFails++ |
| 229 | log.Printf("tunnel: %s re-probe failed (%d): %v", host, probeFails, err) |
| 230 | if probeFails >= 2 { |
| 231 | writeJSON(ctx, c, ControlMsg{Type: "error", Error: "the tunnelled backend stopped looking like an Eleven instance"}) |
| 232 | done = true |
| 233 | } |
| 234 | } else { |
| 235 | probeFails = 0 |
| 236 | } |
| 237 | case <-ageOut.C: |
| 238 | writeJSON(ctx, c, ControlMsg{Type: "error", Error: "trial tunnel expired (24h) — run lchat try again for a fresh address"}) |
| 239 | done = true |
| 240 | case <-t.closed: |
| 241 | done = true |
| 242 | case <-ctx.Done(): |
| 243 | done = true |
| 244 | } |
| 245 | } |
| 246 | g.drop(t) |
| 247 | log.Printf("tunnel: %s closed", host) |
| 248 | } |
| 249 | |
| 250 | // handleConn is the client dialling back with the data connection for one |
| 251 | // pending id; it becomes a raw byte pipe paired with a waiting proxy dial. |
| 252 | func (g *Gateway) handleConn(w http.ResponseWriter, r *http.Request) { |
| 253 | id := strings.TrimPrefix(r.URL.Path, "/api/tunnel/conn/") |
| 254 | g.mu.Lock() |
| 255 | ch := g.pending[id] |
| 256 | delete(g.pending, id) |
| 257 | g.mu.Unlock() |
| 258 | if ch == nil { |
| 259 | http.Error(w, "unknown conn id", http.StatusNotFound) |
| 260 | return |
| 261 | } |
| 262 | c, err := websocket.Accept(w, r, nil) |
| 263 | if err != nil { |
| 264 | close(ch) |
| 265 | return |
| 266 | } |
| 267 | c.SetReadLimit(-1) |
| 268 | // Lifetime belongs to the proxied exchange now, not this handler's request |
| 269 | // context — NetConn with Background keeps it open until either side closes. |
| 270 | conn := websocket.NetConn(context.Background(), c, websocket.MessageBinary) |
| 271 | select { |
| 272 | case ch <- conn: |
| 273 | // The proxy owns it now; block here isn't needed — Accept hijacked the |
| 274 | // HTTP connection, and returning is fine. |
| 275 | case <-time.After(5 * time.Second): |
| 276 | conn.Close() |
| 277 | } |
| 278 | } |
| 279 | |
| 280 | // proxyFor builds the reverse proxy whose transport dials the client through |
| 281 | // the control channel: every new backend connection is one "dial" message and |
| 282 | // one dialled-back WebSocket. |
| 283 | func (g *Gateway) proxyFor(t *liveTunnel) *httputil.ReverseProxy { |
| 284 | // A browser's first paint of the no-build shell is a burst of 40+ module |
| 285 | // fetches over HTTP/2 — wider than the per-tunnel dial-back cap. Bounding |
| 286 | // the transport just under the cap makes the overflow queue for a free |
| 287 | // connection (as browsers always did against HTTP/1.1's 6-connection |
| 288 | // limit) instead of erroring into 502s, any one of which permanently |
| 289 | // fails the page's ES module graph. Two slots stay free for the gateway's |
| 290 | // own liveness probes, which dial back outside this transport; the idle |
| 291 | // pool matches so a finished request hands its connection to a queued one. |
| 292 | perHost := max(1, g.maxConns()-2) |
| 293 | transport := &http.Transport{ |
| 294 | DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| 295 | return g.dialBack(ctx, t) |
| 296 | }, |
| 297 | MaxConnsPerHost: perHost, |
| 298 | MaxIdleConns: perHost, |
| 299 | IdleConnTimeout: 60 * time.Second, |
| 300 | MaxIdleConnsPerHost: perHost, |
| 301 | } |
| 302 | return &httputil.ReverseProxy{ |
| 303 | Rewrite: func(pr *httputil.ProxyRequest) { |
| 304 | pr.SetXForwarded() |
| 305 | pr.Out.URL.Scheme = "http" |
| 306 | pr.Out.URL.Host = "tunnel-local" |
| 307 | pr.Out.Host = pr.In.Host // the local instance sees its public hostname |
| 308 | }, |
| 309 | Transport: transport, |
| 310 | ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { |
| 311 | http.Error(w, "the trial space didn't answer — is lchat try still running?", http.StatusBadGateway) |
| 312 | }, |
| 313 | } |
| 314 | } |
| 315 | |
| 316 | func (g *Gateway) dialBack(ctx context.Context, t *liveTunnel) (net.Conn, error) { |
| 317 | if int(t.conns.Add(1)) > g.maxConns() { |
| 318 | t.conns.Add(-1) |
| 319 | return nil, fmt.Errorf("tunnel connection cap reached") |
| 320 | } |
| 321 | release := func() { t.conns.Add(-1) } |
| 322 | id, err := randToken(18) |
| 323 | if err != nil { |
| 324 | release() |
| 325 | return nil, err |
| 326 | } |
| 327 | ch := make(chan net.Conn, 1) |
| 328 | g.mu.Lock() |
| 329 | g.pending[id] = ch |
| 330 | g.mu.Unlock() |
| 331 | cleanup := func() { |
| 332 | g.mu.Lock() |
| 333 | delete(g.pending, id) |
| 334 | g.mu.Unlock() |
| 335 | } |
| 336 | // The dial message rides the shared control channel, so its write is |
| 337 | // bounded by the tunnel's own life, never by one visitor's request ctx: |
| 338 | // the websocket library abandons the whole connection when a write is |
| 339 | // cancelled mid-frame, which would let a single reload mid-load kill the |
| 340 | // trial for everyone (writeJSON supplies its own 10s bound). A write |
| 341 | // error therefore really does mean the control channel is dead. |
| 342 | if err := writeJSON(context.Background(), t.control, ControlMsg{Type: "dial", ID: id}); err != nil { |
| 343 | cleanup() |
| 344 | release() |
| 345 | g.drop(t) // control channel is dead — the tunnel is gone |
| 346 | return nil, fmt.Errorf("tunnel control write: %w", err) |
| 347 | } |
| 348 | select { |
| 349 | case conn, ok := <-ch: |
| 350 | if !ok || conn == nil { |
| 351 | cleanup() |
| 352 | release() |
| 353 | return nil, fmt.Errorf("tunnel dial-back failed") |
| 354 | } |
| 355 | return &countedConn{Conn: conn, release: release}, nil |
| 356 | case <-time.After(g.dialTimeout()): |
| 357 | cleanup() |
| 358 | release() |
| 359 | return nil, fmt.Errorf("tunnel dial-back timed out") |
| 360 | case <-ctx.Done(): |
| 361 | cleanup() |
| 362 | release() |
| 363 | return nil, ctx.Err() |
| 364 | } |
| 365 | } |
| 366 | |
| 367 | // countedConn releases its tunnel's connection slot exactly once on close. |
| 368 | type countedConn struct { |
| 369 | net.Conn |
| 370 | release func() |
| 371 | once sync.Once |
| 372 | } |
| 373 | |
| 374 | func (c *countedConn) Close() error { |
| 375 | c.once.Do(c.release) |
| 376 | return c.Conn.Close() |
| 377 | } |
| 378 | |
| 379 | // probeBackend asks the tunnelled backend to prove it's an lchat instance |
| 380 | // before (and periodically after) its hostname goes public: GET /api/version |
| 381 | // through the tunnel must return 200 and a version. This keeps the trial |
| 382 | // domain lending its name — and its TLS — only to actual Eleven instances, |
| 383 | // not to whatever a modified client points --local at. |
| 384 | func (g *Gateway) probeBackend(ctx context.Context, t *liveTunnel) error { |
| 385 | client := &http.Client{ |
| 386 | Timeout: 10 * time.Second, |
| 387 | Transport: &http.Transport{ |
| 388 | DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| 389 | return g.dialBack(ctx, t) |
| 390 | }, |
| 391 | }, |
| 392 | } |
| 393 | req, err := http.NewRequestWithContext(ctx, "GET", "http://tunnel-local/api/version", nil) |
| 394 | if err != nil { |
| 395 | return err |
| 396 | } |
| 397 | req.Host = t.host |
| 398 | resp, err := client.Do(req) |
| 399 | if err != nil { |
| 400 | return fmt.Errorf("backend didn't answer: %w", err) |
| 401 | } |
| 402 | defer resp.Body.Close() |
| 403 | if resp.StatusCode != http.StatusOK { |
| 404 | return fmt.Errorf("backend /api/version: HTTP %d", resp.StatusCode) |
| 405 | } |
| 406 | var v struct { |
| 407 | Version string `json:"version"` |
| 408 | } |
| 409 | if err := json.NewDecoder(resp.Body).Decode(&v); err != nil || v.Version == "" { |
| 410 | return fmt.Errorf("backend isn't an Eleven instance") |
| 411 | } |
| 412 | return nil |
| 413 | } |
| 414 | |
| 415 | // drop removes the tunnel everywhere, once. The map guard matters for |
| 416 | // relays: a superseded tunnel dropping late must not take down the |
| 417 | // reattached one that now owns the hostname. |
| 418 | func (g *Gateway) drop(t *liveTunnel) { |
| 419 | t.once.Do(func() { |
| 420 | close(t.closed) |
| 421 | g.mu.Lock() |
| 422 | owns := g.tunnels[t.host] == t |
| 423 | if owns { |
| 424 | delete(g.tunnels, t.host) |
| 425 | } |
| 426 | g.mu.Unlock() |
| 427 | if owns && g.OnClose != nil { |
| 428 | g.OnClose(t.host) |
| 429 | } |
| 430 | t.control.CloseNow() |
| 431 | }) |
| 432 | } |
| 433 | |
| 434 | var errNoFreeHost = fmt.Errorf("couldn't find a free address") |
| 435 | |
| 436 | // admit enforces the per-IP rate and the global cap. Guardrails, not a wall: |
| 437 | // this is a free trial surface on a shared box, and the failure mode we're |
| 438 | // preventing is someone scripting a thousand tunnels, not a family trying it |
| 439 | // twice. |
| 440 | func (g *Gateway) admit(ip string) error { |
| 441 | g.mu.Lock() |
| 442 | defer g.mu.Unlock() |
| 443 | if g.tunnels == nil { |
| 444 | g.tunnels = map[string]*liveTunnel{} |
| 445 | g.pending = map[string]chan net.Conn{} |
| 446 | g.byIP = map[string][]time.Time{} |
| 447 | } |
| 448 | if len(g.tunnels) >= g.maxTunnels() { |
| 449 | return fmt.Errorf("the trial gateway is full right now — please try again later") |
| 450 | } |
| 451 | cutoff := time.Now().Add(-time.Hour) |
| 452 | kept := g.byIP[ip][:0] |
| 453 | for _, ts := range g.byIP[ip] { |
| 454 | if ts.After(cutoff) { |
| 455 | kept = append(kept, ts) |
| 456 | } |
| 457 | } |
| 458 | g.byIP[ip] = kept |
| 459 | if len(kept) >= g.maxPerIP() { |
| 460 | return fmt.Errorf("too many trial spaces from this address — please try again in an hour") |
| 461 | } |
| 462 | return nil |
| 463 | } |
| 464 | |
| 465 | func (g *Gateway) newHost() (string, error) { |
| 466 | for range 20 { |
| 467 | slug := make([]byte, 5) |
| 468 | for i := range slug { |
| 469 | n, err := rand.Int(rand.Reader, big.NewInt(int64(len(slugAlphabet)))) |
| 470 | if err != nil { |
| 471 | return "", err |
| 472 | } |
| 473 | slug[i] = slugAlphabet[n.Int64()] |
| 474 | } |
| 475 | host := "try-" + string(slug) + "." + g.Domain |
| 476 | g.mu.Lock() |
| 477 | _, taken := g.tunnels[host] |
| 478 | g.mu.Unlock() |
| 479 | if !taken { |
| 480 | return host, nil |
| 481 | } |
| 482 | } |
| 483 | return "", fmt.Errorf("couldn't find a free address") |
| 484 | } |
| 485 | |
| 486 | // clientIP prefers the router's X-Forwarded-For (it fronts us over a unix |
| 487 | // socket, so RemoteAddr is meaningless). |
| 488 | func clientIP(r *http.Request) string { |
| 489 | if xff := r.Header.Get("X-Forwarded-For"); xff != "" { |
| 490 | return strings.TrimSpace(strings.Split(xff, ",")[0]) |
| 491 | } |
| 492 | if h, _, err := net.SplitHostPort(r.RemoteAddr); err == nil { |
| 493 | return h |
| 494 | } |
| 495 | return r.RemoteAddr |
| 496 | } |
| 497 | |
| 498 | func randToken(n int) (string, error) { |
| 499 | b := make([]byte, n) |
| 500 | if _, err := rand.Read(b); err != nil { |
| 501 | return "", err |
| 502 | } |
| 503 | return base64.RawURLEncoding.EncodeToString(b), nil |
| 504 | } |
| 505 | |
| 506 | func readJSON(ctx context.Context, c *websocket.Conn, v any) error { |
| 507 | _, data, err := c.Read(ctx) |
| 508 | if err != nil { |
| 509 | return err |
| 510 | } |
| 511 | return json.Unmarshal(data, v) |
| 512 | } |
| 513 | |
| 514 | func writeJSON(ctx context.Context, c *websocket.Conn, v any) error { |
| 515 | data, err := json.Marshal(v) |
| 516 | if err != nil { |
| 517 | return err |
| 518 | } |
| 519 | wctx, cancel := context.WithTimeout(ctx, 10*time.Second) |
| 520 | defer cancel() |
| 521 | return c.Write(wctx, websocket.MessageText, data) |
| 522 | } |
| 523 | |
amadan