carlos / platform Public

feat: edge — host router with unix and TCP upstreams, ACME, live reload

Ported from Eleven router.go (identity-free shape) with titogo's
watcher fixes; TCP Addr upstreams added for whole-app tenants.
infra: tito skeleton — ie region, buckets, svc-pdf, static DNS.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell 6f8bb910c908018b112474d25078e838bcb19d47 parent f374052
12 files changed, +1126 −3
  • go.mod +1 −1
  • infra/tito/dns.tf +18 −0
  • infra/tito/main.tf +84 −0
  • infra/tito/outputs.tf +15 −0
  • infra/tito/region-ie.tf +116 −0
  • infra/tito/svc-pdf-init.sh +44 −0
  • infra/tito/svc-pdf.tf +60 −0
  • internal/edge/certcache.go +34 −0
  • internal/edge/edge.go +449 −0
  • internal/edge/edge_test.go +281 −0
  • internal/edge/static.go +20 −0
  • main.go +4 −2
diff --git a/go.mod b/go.mod
index 22ac241e8..642039770 100644
--- a/go.mod
+++ b/go.mod
@@ -6,6 +6,7 @@ require (
github.com/aws/aws-sdk-go-v2 v1.42.1
github.com/aws/aws-sdk-go-v2/config v1.32.6
github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0
+ github.com/aws/smithy-go v1.27.3
github.com/benbjohnson/litestream v0.5.14
golang.org/x/crypto v0.52.0
modernc.org/sqlite v1.55.0
@@ -28,7 +29,6 @@ require (
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
- github.com/aws/smithy-go v1.27.3 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
diff --git a/infra/tito/dns.tf b/infra/tito/dns.tf
new file mode 100644
index 000000000..c842e92db
--- /dev/null
+++ b/infra/tito/dns.tf
@@ -0,0 +1,18 @@
+# Static DNS skeleton in the existing tito.io Route53 zone. ONLY the
+# regional alias is created here — go.tito.io and *.go.tito.io keep
+# pointing at the Hetzner box until cutover (Tito infra spec §12), and
+# the cutover flip is deliberately NOT a tofu change (it is the
+# platform/operator's dynamic-DNS territory; nothing account-shaped in
+# tofu state).
+
+data "aws_route53_zone" "tito" {
+ name = "tito.io."
+}
+
+resource "aws_route53_record" "ie_alias" {
+ zone_id = data.aws_route53_zone.tito.zone_id
+ name = "ie.go.tito.io"
+ type = "A"
+ ttl = 300
+ records = [aws_eip.host_ie.public_ip]
+}
diff --git a/infra/tito/main.tf b/infra/tito/main.tf
new file mode 100644
index 000000000..db38e9d74
--- /dev/null
+++ b/infra/tito/main.tf
@@ -0,0 +1,84 @@
+# Tito's self-hosted CARLOS deployment — AWS account 665106592142 (Tito's
+# own; platform spec §12, Tito infra spec). Tofu owns boxes, buckets, IAM,
+# and static DNS; the platform owns apps, releases, instances, and dynamic
+# DNS. Nothing account-shaped may ever enter this state.
+
+terraform {
+ required_version = ">= 1.10"
+ backend "s3" {
+ bucket = "carlos-tfstate-665106592142"
+ key = "tito/terraform.tfstate"
+ region = "eu-west-1"
+ profile = "titogo"
+ use_lockfile = true
+ }
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "~> 6.0"
+ }
+ }
+}
+
+provider "aws" {
+ region = "eu-west-1"
+ profile = "titogo"
+}
+
+locals {
+ account = "665106592142"
+}
+
+# The deployment bucket: apps, releases, channels, instance records, host
+# config. Routing metadata only — nothing customer-shaped rests here.
+resource "aws_s3_bucket" "deployment" {
+ bucket = "carlos-tito-platform-${local.account}"
+ lifecycle { prevent_destroy = true }
+}
+
+# The IE regional data bucket: parked databases, WAL streams, leases,
+# blobs. This bucket is the residency promise (Tito infra spec §4).
+resource "aws_s3_bucket" "data_ie" {
+ bucket = "carlos-tito-ie-${local.account}"
+ lifecycle { prevent_destroy = true }
+}
+
+resource "aws_s3_bucket_versioning" "deployment" {
+ bucket = aws_s3_bucket.deployment.id
+ versioning_configuration { status = "Enabled" }
+}
+
+resource "aws_s3_bucket_versioning" "data_ie" {
+ bucket = aws_s3_bucket.data_ie.id
+ versioning_configuration { status = "Enabled" }
+}
+
+resource "aws_s3_bucket_public_access_block" "deployment" {
+ bucket = aws_s3_bucket.deployment.id
+ block_public_acls = true
+ ignore_public_acls = true
+ block_public_policy = true
+ restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_public_access_block" "data_ie" {
+ bucket = aws_s3_bucket.data_ie.id
+ block_public_acls = true
+ ignore_public_acls = true
+ block_public_policy = true
+ restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "deployment" {
+ bucket = aws_s3_bucket.deployment.id
+ rule {
+ apply_server_side_encryption_by_default { sse_algorithm = "AES256" }
+ }
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "data_ie" {
+ bucket = aws_s3_bucket.data_ie.id
+ rule {
+ apply_server_side_encryption_by_default { sse_algorithm = "AES256" }
+ }
+}
diff --git a/infra/tito/outputs.tf b/infra/tito/outputs.tf
new file mode 100644
index 000000000..ab83c78fb
--- /dev/null
+++ b/infra/tito/outputs.tf
@@ -0,0 +1,15 @@
+output "ie_eip" {
+ value = aws_eip.host_ie.public_ip
+}
+
+output "deployment_bucket" {
+ value = aws_s3_bucket.deployment.bucket
+}
+
+output "data_bucket_ie" {
+ value = aws_s3_bucket.data_ie.bucket
+}
+
+output "svc_pdf_private_ip" {
+ value = aws_instance.svc_pdf.private_ip
+}
diff --git a/infra/tito/region-ie.tf b/infra/tito/region-ie.tf
new file mode 100644
index 000000000..587c0af61
--- /dev/null
+++ b/infra/tito/region-ie.tf
@@ -0,0 +1,116 @@
+# The IE region of Tito's deployment: edge/control-plane home (Tito infra
+# spec §2.2). One Graviton box running `carlos agent`. t4g.medium — the
+# Hetzner fleet's whole workload fits with headroom (its redline was 3.8GB
+# with ~20 instances at ~190MB).
+
+data "aws_ami" "debian_arm64" {
+ most_recent = true
+ owners = ["136693071363"] # Debian
+ filter {
+ name = "name"
+ values = ["debian-13-arm64-*"]
+ }
+ filter {
+ name = "architecture"
+ values = ["arm64"]
+ }
+}
+
+resource "aws_security_group" "host_ie" {
+ name = "carlos-tito-ie"
+ description = "carlos tito IE host"
+ ingress {
+ from_port = 80
+ to_port = 80
+ protocol = "tcp"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+ ingress {
+ from_port = 443
+ to_port = 443
+ protocol = "tcp"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+ # No SSH ingress: admin access is SSM Session Manager only.
+ egress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+}
+
+resource "aws_iam_role" "host_ie" {
+ name = "carlos-tito-host-ie"
+ assume_role_policy = jsonencode({
+ Version = "2012-10-17"
+ Statement = [{
+ Effect = "Allow"
+ Principal = { Service = "ec2.amazonaws.com" }
+ Action = "sts:AssumeRole"
+ }]
+ })
+}
+
+# Residency enforced by construction: this role reaches the deployment
+# bucket plus the IE data bucket and nothing else. A future Sydney host's
+# role will not include the IE bucket — the wrong-region wake the spec
+# requires to fail loudly fails at the IAM layer.
+resource "aws_iam_role_policy" "host_ie_buckets" {
+ name = "carlos-tito-host-ie-buckets"
+ role = aws_iam_role.host_ie.id
+ policy = jsonencode({
+ Version = "2012-10-17"
+ Statement = [
+ {
+ Effect = "Allow"
+ Action = ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"]
+ Resource = [
+ "${aws_s3_bucket.deployment.arn}/*",
+ "${aws_s3_bucket.data_ie.arn}/*",
+ ]
+ },
+ {
+ Effect = "Allow"
+ Action = ["s3:ListBucket", "s3:GetBucketLocation"]
+ Resource = [aws_s3_bucket.deployment.arn, aws_s3_bucket.data_ie.arn]
+ },
+ {
+ Effect = "Allow"
+ Action = ["ssm:GetParameter"]
+ Resource = "arn:aws:ssm:eu-west-1:${local.account}:parameter/carlos/*"
+ },
+ ]
+ })
+}
+
+resource "aws_iam_role_policy_attachment" "host_ie_ssm" {
+ role = aws_iam_role.host_ie.name
+ policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+}
+
+resource "aws_iam_instance_profile" "host_ie" {
+ name = "carlos-tito-host-ie"
+ role = aws_iam_role.host_ie.name
+}
+
+resource "aws_instance" "host_ie" {
+ ami = data.aws_ami.debian_arm64.id
+ instance_type = "t4g.medium"
+ vpc_security_group_ids = [aws_security_group.host_ie.id]
+ iam_instance_profile = aws_iam_instance_profile.host_ie.name
+ user_data = file("${path.module}/host-init.sh")
+ user_data_replace_on_change = true
+ root_block_device {
+ volume_size = 40
+ volume_type = "gp3"
+ encrypted = true
+ }
+ tags = { Name = "carlos-tito-ie" }
+}
+
+resource "aws_eip" "host_ie" {
+ domain = "vpc"
+ instance = aws_instance.host_ie.id
+ tags = { Name = "carlos-tito-ie" }
+}
diff --git a/infra/tito/svc-pdf-init.sh b/infra/tito/svc-pdf-init.sh
new file mode 100644
index 000000000..498376478
--- /dev/null
+++ b/infra/tito/svc-pdf-init.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+# 🤖 First-boot init for svc/pdf: Gotenberg (pinned image) in Docker,
+# loopback-only, fronted by Caddy. Stateless by construction — this box
+# has no bucket access, no database, and nothing worth backing up.
+set -euo pipefail
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -q
+apt-get install -qy docker.io curl unattended-upgrades
+systemctl enable --now unattended-upgrades docker
+
+# Admin access is SSM Session Manager only.
+curl -fsSL -o /tmp/amazon-ssm-agent.deb \
+ "https://s3.eu-west-1.amazonaws.com/amazon-ssm-eu-west-1/latest/debian_arm64/amazon-ssm-agent.deb"
+dpkg -i /tmp/amazon-ssm-agent.deb
+systemctl enable --now amazon-ssm-agent
+
+# Gotenberg, pinned, loopback only — Caddy is the only way in.
+docker run -d --name gotenberg --restart=always -p 127.0.0.1:3000:3000 \
+ gotenberg/gotenberg:8.21.1
+
+# Caddy terminates TLS on the instance's internal DNS name with a
+# self-signed cert (fleet-internal traffic; the caller pins basic-auth,
+# not the cert). Basic-auth credentials arrive via SSM at first use —
+# the operator sets /etc/caddy/pdf.env through a session.
+apt-get install -qy caddy
+cat >/etc/caddy/Caddyfile <<'EOF'
+{
+ auto_https disable_redirects
+}
+:443 {
+ tls internal
+ basic_auth {
+ import /etc/caddy/pdf-basicauth
+ }
+ reverse_proxy 127.0.0.1:3000
+}
+EOF
+# Placeholder until the operator writes the real bcrypt line via SSM:
+# format: "<user> <bcrypt-hash>"
+touch /etc/caddy/pdf-basicauth
+systemctl restart caddy
+
+echo "svc/pdf ready — set /etc/caddy/pdf-basicauth via SSM" >/etc/motd
diff --git a/infra/tito/svc-pdf.tf b/infra/tito/svc-pdf.tf
new file mode 100644
index 000000000..72ba0ae77
--- /dev/null
+++ b/infra/tito/svc-pdf.tf
@@ -0,0 +1,60 @@
+# svc/pdf: one Gotenberg behind Caddy basic-auth — the platform spec's
+# PDF/render service (seeing, therefore stateless BY CONSTRUCTION: no
+# bucket grant, no database, no volume beyond the OS). Tito's invoice
+# work can consume this before any platform code exists.
+
+resource "aws_security_group" "svc_pdf" {
+ name = "carlos-tito-svc-pdf"
+ description = "Gotenberg — reachable from the fleet only"
+ ingress {
+ from_port = 443
+ to_port = 443
+ protocol = "tcp"
+ security_groups = [aws_security_group.host_ie.id]
+ }
+ egress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+}
+
+# SSM only; NO bucket or SSM-parameter grants — statelessness is imposed,
+# not promised.
+resource "aws_iam_role" "svc_pdf" {
+ name = "carlos-tito-svc-pdf"
+ assume_role_policy = jsonencode({
+ Version = "2012-10-17"
+ Statement = [{
+ Effect = "Allow"
+ Principal = { Service = "ec2.amazonaws.com" }
+ Action = "sts:AssumeRole"
+ }]
+ })
+}
+
+resource "aws_iam_role_policy_attachment" "svc_pdf_ssm" {
+ role = aws_iam_role.svc_pdf.name
+ policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+}
+
+resource "aws_iam_instance_profile" "svc_pdf" {
+ name = "carlos-tito-svc-pdf"
+ role = aws_iam_role.svc_pdf.name
+}
+
+resource "aws_instance" "svc_pdf" {
+ ami = data.aws_ami.debian_arm64.id
+ instance_type = "t4g.small"
+ vpc_security_group_ids = [aws_security_group.svc_pdf.id]
+ iam_instance_profile = aws_iam_instance_profile.svc_pdf.name
+ user_data = file("${path.module}/svc-pdf-init.sh")
+ user_data_replace_on_change = true
+ root_block_device {
+ volume_size = 20
+ volume_type = "gp3"
+ encrypted = true
+ }
+ tags = { Name = "carlos-tito-svc-pdf" }
+}
diff --git a/internal/edge/certcache.go b/internal/edge/certcache.go
new file mode 100644
index 000000000..c534d429a
--- /dev/null
+++ b/internal/edge/certcache.go
@@ -0,0 +1,34 @@
+package edge
+
+// Ported from titogo internal/carlos/certcache.go (itself adapted from
+// Eleven's certcache.go): the certificate-cache seam. Edge boxes in a pool
+// must eventually SHARE issued certs, so every issuance artifact goes through
+// one autocert.Cache chosen here, never direct file I/O. autocert already
+// defines exactly the interface a remote backend needs (Get/Put/Delete with
+// ctx, ErrCacheMiss for absent), so there is no parallel abstraction: an S3
+// backend is a drop-in behind this constructor.
+
+import (
+ "fmt"
+ "os"
+ "strings"
+
+ "golang.org/x/crypto/acme/autocert"
+)
+
+// newCertCache returns the certificate cache selected by CARLOS_CERT_CACHE.
+// Unset (or "dir") is the on-disk layout — autocert.DirCache over dir, cache
+// keys as plain file names — so an existing cache keeps working over a
+// restart with no migration. "s3" is reserved for a pooled-edge backend;
+// until it exists it refuses to start, because failing loudly beats silently
+// minting certs into a local dir the rest of a pool can't see.
+func newCertCache(dir string) (autocert.Cache, error) {
+ switch backend := strings.TrimSpace(os.Getenv("CARLOS_CERT_CACHE")); backend {
+ case "", "dir":
+ return autocert.DirCache(dir), nil
+ case "s3":
+ return nil, fmt.Errorf("CARLOS_CERT_CACHE=s3: reserved, not implemented yet")
+ default:
+ return nil, fmt.Errorf("CARLOS_CERT_CACHE=%q: want unset, \"dir\", or \"s3\"", backend)
+ }
+}
diff --git a/internal/edge/edge.go b/internal/edge/edge.go
new file mode 100644
index 000000000..c5018c3de
--- /dev/null
+++ b/internal/edge/edge.go
@@ -0,0 +1,449 @@
+// Package edge is the deployment host's front door: TLS termination
+// (autocert), a host→backend reverse proxy, and a live-reloading route table.
+//
+// Ported from Eleven's router.go (elevenmessenger/messenger: router.go;
+// design in its docs/instance-router.md) — the identity-free shape: the
+// atomically-swapped host→upstream map, the retrying unix dialer that masks
+// instance restarts, autocert with the route table as HostPolicy, the
+// registry fingerprint watcher, SIGHUP reload, the unknown-host fallback, and
+// the hibernation-activator hook — with titogo internal/carlos/router.go's
+// additions carried in: -dev (plain-HTTP mode, so multi-instance routing runs
+// locally with no DNS/TLS) and the first-observed-fingerprint reload. Dropped
+// on port: titogo's Stripe relay, auth/identity, root-domain platformIndex;
+// Eleven's mesh and wildcard DNS-01 certs. New here: TCP upstreams (Route.Addr)
+// so whole-app tenants sit behind the edge without unix-socket support, and
+// the static serving class stub (static.go).
+package edge
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "log"
+ "net"
+ "net/http"
+ "net/http/httputil"
+ "os"
+ "os/signal"
+ "path/filepath"
+ "sync/atomic"
+ "syscall"
+ "time"
+
+ "golang.org/x/crypto/acme/autocert"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// Activator is the hibernation seam (implemented by the activator package):
+// given a hibernate-flagged instance route, it wakes (or joins an in-progress
+// wake of) the backing process before letting the request through, and
+// reports whether it wrote the response. nil = no hibernation, pure proxy.
+type Activator interface {
+ ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool
+ RetireExcept(keep map[string]bool)
+}
+
+// Edge terminates TLS (autocert) and reverse-proxies each configured host to
+// its backend. The host→backend map is swapped atomically, so reloads and
+// zero-downtime deploys never drop a live connection.
+type Edge struct {
+ reg *registry.Registry
+ upstreams atomic.Pointer[map[string]upstream]
+ // act, when non-nil, is the hibernation activator: hibernate-flagged
+ // instance routes stop being always-on backends and become on-demand
+ // processes woken per request. Nil keeps the proxy-only behavior; with it
+ // set, unflagged routes are still proxied straight (the flag is per host).
+ act Activator
+ // fallbackURL, when set, is where requests for unknown hosts are redirected
+ // (e.g. the apex marketing site) instead of a bare 404 — a wildcard DNS
+ // record then makes typo'd or retired subdomains land somewhere useful. Note
+ // the limit: over HTTPS this only helps hosts that still HAVE a cert (e.g. a
+ // removed instance); a never-served host fails the TLS handshake first,
+ // because HostPolicy refuses to mint certs for unknown hosts (anything else
+ // would let strangers burn the ACME rate limit). The :80 path always works.
+ fallbackURL string
+}
+
+// upstream is one served host: the handler for its backend, plus — when the
+// activator manages it — the route itself, so a request can trigger the wake.
+type upstream struct {
+ proxy http.Handler
+ route registry.Route
+ // hibernating marks an activator-managed instance: the edge itself knows
+ // whether its process is live (it's the edge's child), so "dead backend"
+ // detection is the activator's state table, not a failed dial.
+ hibernating bool
+}
+
+// New builds an Edge over reg and performs the startup reload. act may be nil
+// (no hibernation).
+func New(reg *registry.Registry, act Activator) (*Edge, error) {
+ e := &Edge{reg: reg, act: act}
+ if err := e.Reload(); err != nil {
+ return nil, err
+ }
+ return e, nil
+}
+
+// Reload rebuilds the host→backend map from the registry and swaps it in
+// atomically. Safe to call while serving.
+func (e *Edge) Reload() error {
+ routes, err := e.reg.List()
+ if err != nil {
+ return err
+ }
+ m := make(map[string]upstream, len(routes))
+ // The hosts the activator should manage after this reload — feeds both the
+ // per-upstream gate and the retire pass below.
+ managed := make(map[string]bool)
+ for _, t := range routes {
+ // Only serve ready routes: a 'creating'/'failed' entry has no working
+ // backend, so proxying it would just stall → 502 — AND, since Allowed
+ // is the autocert HostPolicy, serving it would burn an ACME issuance
+ // on a host that can't answer yet.
+ if t.Status != "ready" {
+ continue
+ }
+ var proxy http.Handler
+ switch {
+ case t.Socket != "":
+ proxy = unixProxy(t.Socket)
+ case t.Addr != "":
+ proxy = tcpProxy(t.Addr)
+ case t.Kind == "static":
+ proxy = staticHandler(t)
+ default:
+ continue // no backend to reach
+ }
+ // Hibernation is per-host: only an instance whose route carries the
+ // hibernate flag is activator-managed; unflagged instances and sibling
+ // services are proxied straight. With an activator wired but no route
+ // flagged, the edge therefore behaves exactly as it does without one —
+ // so the binary + env can ship fleet-wide first, and the flag flips
+ // one instance at a time.
+ hib := e.act != nil && t.Kind == "instance" && t.Hibernate
+ if hib {
+ managed[t.Host] = true
+ }
+ m[t.Host] = upstream{proxy: proxy, route: t, hibernating: hib}
+ }
+ e.upstreams.Store(&m)
+ // Reconcile the activator with the new route set: a host un-flagged (or
+ // removed) since the last reload may still have a live child holding its
+ // socket and lease — retire it so a systemd unit can take over cleanly.
+ // After the map swap above, so no new request re-enters the activator for
+ // a host being retired.
+ if e.act != nil {
+ e.act.RetireExcept(managed)
+ }
+ return nil
+}
+
+func (e *Edge) current() map[string]upstream {
+ if p := e.upstreams.Load(); p != nil {
+ return *p
+ }
+ return nil
+}
+
+// Allowed reports whether host is configured — used as the autocert
+// HostPolicy, so a certificate is only ever requested for a host we serve.
+func (e *Edge) Allowed(host string) bool {
+ _, ok := e.current()[host]
+ return ok
+}
+
+func (e *Edge) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ host := r.Host
+ if h, _, err := net.SplitHostPort(host); err == nil {
+ host = h
+ }
+ up, ok := e.current()[host]
+ if !ok {
+ e.unknownHost(w, r)
+ return
+ }
+ if up.hibernating {
+ // Wake (or join an in-progress wake) before proxying; the activator
+ // also stamps this request's activity for the idle clock. Note a bare
+ // TLS handshake never wakes anything — only a proxied request does.
+ if e.act.ServeHTTP(w, r, up.route, up.proxy) {
+ return
+ }
+ }
+ up.proxy.ServeHTTP(w, r)
+}
+
+// unknownHost answers a request for a host we don't serve: a redirect to the
+// fallback (302 — a typo'd host must not be cached as permanently moved), or
+// the classic 404 when no fallback is configured.
+func (e *Edge) unknownHost(w http.ResponseWriter, r *http.Request) {
+ if e.fallbackURL != "" {
+ http.Redirect(w, r, e.fallbackURL, http.StatusFound)
+ return
+ }
+ http.Error(w, "unknown host", http.StatusNotFound)
+}
+
+// Watch polls the registry for writes from other processes (`carlos add`
+// etc., without needing rights to signal us) every 2s and reloads on change.
+// It polls a cheap fingerprint of the routes table rather than PRAGMA
+// data_version: the pragma proved blind to cross-process commits on Eleven's
+// production box (a modernc/sqlite WAL-visibility quirk under long-lived
+// connections). registry.Fingerprint opens a FRESH connection per poll — a
+// brand-new connection reads the latest committed state by construction
+// (see registry.List's comment for the stale-WAL-snapshot history). Opening
+// SQLite costs about a millisecond; at one poll every two seconds that is
+// nothing. Returns when stop closes; SIGHUP-forced reloads live in Run.
+func (e *Edge) Watch(stop <-chan struct{}) {
+ log.Printf("edge: registry watcher armed (2s fingerprint poll)")
+ // No baseline-only first tick: the first observed fingerprint also
+ // triggers a reload, so a route added between the edge's startup reload
+ // and this poll can't be silently baselined away. (Eleven's version has
+ // that gap; one redundant reload at startup is the cost of closing it.)
+ var last string
+ errs := 0
+ tick := time.NewTicker(2 * time.Second)
+ defer tick.Stop()
+ for {
+ select {
+ case <-stop:
+ return
+ case <-tick.C:
+ }
+ fp, err := e.reg.Fingerprint()
+ if err != nil {
+ errs++
+ if errs == 1 || errs%300 == 0 {
+ log.Printf("edge: registry-watch poll error (#%d): %v", errs, err)
+ }
+ continue
+ }
+ errs = 0
+ if fp == last {
+ continue
+ }
+ last = fp
+ if err := e.Reload(); err != nil {
+ log.Printf("edge: registry-watch reload failed: %v", err)
+ continue
+ }
+ log.Printf("edge: registry changed — reloaded (%d route(s))", len(e.current()))
+ }
+}
+
+// unixProxy reverse-proxies to a backend listening on a unix socket,
+// preserving the client's Host header (so the backend sees the real hostname)
+// and passing WebSocket upgrades straight through.
+func unixProxy(socketPath string) *httputil.ReverseProxy {
+ dialer := &net.Dialer{}
+ return &httputil.ReverseProxy{
+ Rewrite: func(pr *httputil.ProxyRequest) {
+ pr.SetXForwarded()
+ pr.Out.URL.Scheme = "http"
+ pr.Out.URL.Host = "carlos" // placeholder; the unix dialer ignores it
+ pr.Out.Host = pr.In.Host // backend sees the real hostname
+ },
+ Transport: &http.Transport{
+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+ return dialUnixWithRetry(ctx, dialer, socketPath)
+ },
+ // Bound idle keep-alives so proxies replaced on reload don't leak
+ // their pooled connections + reader goroutines on the long-running
+ // edge.
+ MaxIdleConns: 32,
+ IdleConnTimeout: 90 * time.Second,
+ },
+ }
+}
+
+// tcpProxy reverse-proxies to a TCP upstream at http://addr with the same
+// Host preservation and idle bounds as unixProxy — this is what lets
+// whole-app tenants sit behind the edge without unix-socket support.
+func tcpProxy(addr string) *httputil.ReverseProxy {
+ return &httputil.ReverseProxy{
+ Rewrite: func(pr *httputil.ProxyRequest) {
+ pr.SetXForwarded()
+ pr.Out.URL.Scheme = "http"
+ pr.Out.URL.Host = addr
+ pr.Out.Host = pr.In.Host // backend sees the real hostname
+ },
+ Transport: &http.Transport{
+ MaxIdleConns: 32,
+ IdleConnTimeout: 90 * time.Second,
+ },
+ }
+}
+
+// dialUnixWithRetry masks the brief gap during an instance restart: while the
+// old process has exited and the new one hasn't rebound the socket, dials fail
+// with ENOENT/ECONNREFUSED. Retry those for a short window so no in-flight
+// request is dropped; any other error fails fast. The window must exceed a
+// deploy's socket-absent gap — the instance drain (10s) plus rebind — so an
+// in-flight request is never 502'd mid-deploy; a sustained-down backend hangs
+// this long before 502 (rare: systemd Restart=always makes "down" transient).
+func dialUnixWithRetry(ctx context.Context, d *net.Dialer, socketPath string) (net.Conn, error) {
+ const window = 13 * time.Second
+ deadline := time.Now().Add(window)
+ for {
+ conn, err := d.DialContext(ctx, "unix", socketPath)
+ if err == nil {
+ return conn, nil
+ }
+ if ctx.Err() != nil || !isRetryableDialErr(err) || time.Now().After(deadline) {
+ return nil, err
+ }
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-time.After(100 * time.Millisecond):
+ }
+ }
+}
+
+func isRetryableDialErr(err error) bool {
+ return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ENOENT)
+}
+
+// Run is the `carlos edge` verb: the shared front door for every instance,
+// sibling service, and static site on the box.
+func Run(args []string) {
+ dataDir := envOr("CARLOS_DATA_DIR", "/data")
+ fs := flag.NewFlagSet("edge", flag.ExitOnError)
+ registryPath := fs.String("registry", filepath.Join(dataDir, "registry.db"), "registry SQLite path (default $CARLOS_DATA_DIR/registry.db)")
+ acmeCache := fs.String("acme-cache", filepath.Join(dataDir, "acme"), "autocert certificate cache dir (default $CARLOS_DATA_DIR/acme)")
+ httpsAddr := fs.String("https", ":443", "HTTPS listen address")
+ httpAddr := fs.String("http", ":80", "HTTP listen address (ACME HTTP-01 challenge + redirect to https; in -dev mode, the only listener)")
+ fallbackURL := fs.String("fallback", "", "redirect requests for unknown hosts here, e.g. an apex marketing site; empty keeps the plain 404")
+ dev := fs.Bool("dev", false, "plain-HTTP mode: serve routes on -http with no TLS/ACME (local development only)")
+ _ = fs.Parse(args)
+
+ reg, err := registry.Open(*registryPath)
+ if err != nil {
+ log.Fatalf("edge: open registry %s: %v", *registryPath, err)
+ }
+
+ // No activator yet: `carlos edge` is the pure-proxy role; the hibernation
+ // activator arrives with `carlos agent` (edge + activator in one process).
+ e, err := New(reg, nil)
+ if err != nil {
+ log.Fatalf("edge: load routes: %v", err)
+ }
+ e.fallbackURL = *fallbackURL
+
+ go e.Watch(make(chan struct{})) // process-lifetime; never stopped
+
+ // Reload routes on SIGHUP too (an operator's explicit "now").
+ sighup := make(chan os.Signal, 1)
+ signal.Notify(sighup, syscall.SIGHUP)
+ go func() {
+ for range sighup {
+ if err := e.Reload(); err != nil {
+ log.Printf("edge: reload failed: %v", err)
+ continue
+ }
+ log.Printf("edge: reloaded (%d route(s))", len(e.current()))
+ }
+ }()
+
+ stop := make(chan os.Signal, 1)
+ signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
+
+ if *dev {
+ // ReadHeaderTimeout guards against Slowloris-style slow-header
+ // connection exhaustion. No ReadTimeout/WriteTimeout — those would
+ // break long-lived proxied WebSockets.
+ srv := &http.Server{Addr: *httpAddr, Handler: e, ReadHeaderTimeout: 10 * time.Second}
+ go func() {
+ log.Printf("edge: DEV MODE — plain HTTP, serving %d route(s) on %s (registry %s)",
+ len(e.current()), *httpAddr, *registryPath)
+ if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ log.Fatalf("edge: http listener %s: %v", *httpAddr, err)
+ }
+ }()
+ <-stop
+ log.Printf("edge: draining and shutting down")
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _ = srv.Shutdown(ctx)
+ return
+ }
+
+ certCache, err := newCertCache(*acmeCache)
+ if err != nil {
+ log.Fatalf("edge: %v", err)
+ }
+
+ m := &autocert.Manager{
+ Prompt: autocert.AcceptTOS,
+ Cache: certCache,
+ HostPolicy: func(_ context.Context, host string) error {
+ if e.Allowed(host) {
+ return nil
+ }
+ return fmt.Errorf("acme: host %q not in registry", host)
+ },
+ }
+ // Per-host HTTP-01 only for now. When per-tenant subdomains outgrow the
+ // Let's Encrypt per-domain rate limit (~50 new certs/week), Eleven's
+ // wildcard.go (ACME DNS-01) drops in here, wrapping tlsCfg.GetCertificate.
+ tlsCfg := m.TLSConfig()
+
+ // The non-ACME :80 handler. Known hosts get the classic redirect to https
+ // (GET/HEAD only, 302); unknown hosts go to the fallback URL instead of a
+ // redirect into a doomed TLS handshake (no cert will ever exist for them).
+ httpFallback := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ host := r.Host
+ if h, _, err := net.SplitHostPort(host); err == nil {
+ host = h
+ }
+ if !e.Allowed(host) {
+ e.unknownHost(w, r)
+ return
+ }
+ if r.Method != "GET" && r.Method != "HEAD" {
+ http.Error(w, "Use HTTPS", http.StatusBadRequest)
+ return
+ }
+ http.Redirect(w, r, "https://"+host+r.URL.RequestURI(), http.StatusFound)
+ })
+
+ // ReadHeaderTimeout guards the public front door against Slowloris-style
+ // slow-header connection exhaustion. No ReadTimeout/WriteTimeout — those
+ // would break long-lived proxied WebSockets.
+ httpSrv := &http.Server{Addr: *httpAddr, Handler: m.HTTPHandler(httpFallback), ReadHeaderTimeout: 10 * time.Second}
+ httpsSrv := &http.Server{Addr: *httpsAddr, Handler: e, TLSConfig: tlsCfg, ReadHeaderTimeout: 10 * time.Second}
+
+ // :80 serves the ACME HTTP-01 challenge and redirects everything else.
+ go func() {
+ if err := httpSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ log.Fatalf("edge: http listener %s: %v", *httpAddr, err)
+ }
+ }()
+ go func() {
+ log.Printf("edge: serving %d route(s) on %s (registry %s, acme cache %s)",
+ len(e.current()), *httpsAddr, *registryPath, *acmeCache)
+ if err := httpsSrv.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed {
+ log.Fatalf("edge: https listener %s: %v", *httpsAddr, err)
+ }
+ }()
+
+ // Graceful shutdown on SIGTERM/SIGINT (e.g. an edge binary upgrade): stop
+ // accepting and drain in-flight HTTP before exiting. Proxied WebSockets are
+ // hijacked (not drained); they reconnect through the new edge.
+ <-stop
+ log.Printf("edge: draining and shutting down")
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _ = httpsSrv.Shutdown(ctx)
+ _ = httpSrv.Shutdown(ctx)
+}
+
+func envOr(key, fallback string) string {
+ if v := os.Getenv(key); v != "" {
+ return v
+ }
+ return fallback
+}
diff --git a/internal/edge/edge_test.go b/internal/edge/edge_test.go
new file mode 100644
index 000000000..65bbb5a73
--- /dev/null
+++ b/internal/edge/edge_test.go
@@ -0,0 +1,281 @@
+package edge
+
+import (
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+func openReg(t *testing.T) *registry.Registry {
+ t.Helper()
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatalf("open registry: %v", err)
+ }
+ return reg
+}
+
+// hostEchoBackend is an upstream that records the Host header it saw and
+// answers a fixed body — enough to prove both routing and Host preservation.
+func hostEchoBackend(t *testing.T, body string) (*httptest.Server, *string) {
+ t.Helper()
+ seenHost := new(string)
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ *seenHost = r.Host
+ io.WriteString(w, body)
+ }))
+ t.Cleanup(ts.Close)
+ return ts, seenHost
+}
+
+// getHost GETs the edge server with an explicit Host header, the way a real
+// client reaches a routed hostname.
+func getHost(t *testing.T, edgeURL, host string) (int, string) {
+ t.Helper()
+ req, err := http.NewRequest("GET", edgeURL+"/", nil)
+ if err != nil {
+ t.Fatalf("new request: %v", err)
+ }
+ req.Host = host
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("GET %s (host %s): %v", edgeURL, host, err)
+ }
+ defer resp.Body.Close()
+ b, err := io.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatalf("read body: %v", err)
+ }
+ return resp.StatusCode, string(b)
+}
+
+func TestRoutesHostToTCPUpstream(t *testing.T) {
+ backend, seenHost := hostEchoBackend(t, "hello from backend")
+ addr := backend.Listener.Addr().String()
+
+ reg := openReg(t)
+ if err := reg.Upsert(registry.Route{Host: "app.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+
+ e, err := New(reg, nil)
+ if err != nil {
+ t.Fatalf("New: %v", err)
+ }
+ srv := httptest.NewServer(e)
+ defer srv.Close()
+
+ code, body := getHost(t, srv.URL, "app.example.com")
+ if code != http.StatusOK {
+ t.Fatalf("status = %d, want 200", code)
+ }
+ if body != "hello from backend" {
+ t.Fatalf("body = %q, want backend body", body)
+ }
+ if *seenHost != "app.example.com" {
+ t.Fatalf("backend saw Host %q, want app.example.com (inbound Host must be preserved)", *seenHost)
+ }
+ if !e.Allowed("app.example.com") {
+ t.Fatalf("Allowed(app.example.com) = false, want true")
+ }
+}
+
+func TestSkipsNotReadyRoutes(t *testing.T) {
+ backend, _ := hostEchoBackend(t, "should never be reached")
+ addr := backend.Listener.Addr().String()
+
+ reg := openReg(t)
+ if err := reg.Upsert(registry.Route{Host: "pending.example.com", Kind: "service", Addr: addr, Status: "pending"}); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+
+ e, err := New(reg, nil)
+ if err != nil {
+ t.Fatalf("New: %v", err)
+ }
+ srv := httptest.NewServer(e)
+ defer srv.Close()
+
+ code, _ := getHost(t, srv.URL, "pending.example.com")
+ if code != http.StatusNotFound {
+ t.Fatalf("status = %d, want 404 for a not-ready route", code)
+ }
+ // Allowed feeds the autocert HostPolicy: a not-ready route must not mint
+ // a certificate either.
+ if e.Allowed("pending.example.com") {
+ t.Fatalf("Allowed(pending.example.com) = true, want false")
+ }
+}
+
+func TestReloadPicksUpNewRoute(t *testing.T) {
+ backend, _ := hostEchoBackend(t, "late arrival")
+ addr := backend.Listener.Addr().String()
+
+ reg := openReg(t)
+ e, err := New(reg, nil)
+ if err != nil {
+ t.Fatalf("New: %v", err)
+ }
+ srv := httptest.NewServer(e)
+ defer srv.Close()
+
+ if code, _ := getHost(t, srv.URL, "new.example.com"); code != http.StatusNotFound {
+ t.Fatalf("pre-reload status = %d, want 404", code)
+ }
+
+ // Write via a SECOND handle on the same file — the cross-process shape
+ // (`carlos add` while the edge runs) that exposed the stale-WAL quirk.
+ reg2, err := registry.Open(reg.Path())
+ if err != nil {
+ t.Fatalf("open second handle: %v", err)
+ }
+ if err := reg2.Upsert(registry.Route{Host: "new.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil {
+ t.Fatalf("upsert via second handle: %v", err)
+ }
+
+ if err := e.Reload(); err != nil {
+ t.Fatalf("Reload: %v", err)
+ }
+ code, body := getHost(t, srv.URL, "new.example.com")
+ if code != http.StatusOK || body != "late arrival" {
+ t.Fatalf("post-reload = %d %q, want 200 %q", code, body, "late arrival")
+ }
+}
+
+func TestUnixSocketUpstream(t *testing.T) {
+ sock := filepath.Join(t.TempDir(), "u.sock")
+ ln, err := net.Listen("unix", sock)
+ if err != nil {
+ t.Fatalf("listen unix: %v", err)
+ }
+ seenHost := new(string)
+ backend := &httptest.Server{
+ Listener: ln,
+ Config: &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ *seenHost = r.Host
+ io.WriteString(w, "over the socket")
+ })},
+ }
+ backend.Start()
+ defer backend.Close()
+
+ reg := openReg(t)
+ if err := reg.Upsert(registry.Route{Host: "sock.example.com", Kind: "instance", Socket: sock, Status: "ready"}); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+
+ e, err := New(reg, nil)
+ if err != nil {
+ t.Fatalf("New: %v", err)
+ }
+ srv := httptest.NewServer(e)
+ defer srv.Close()
+
+ code, body := getHost(t, srv.URL, "sock.example.com")
+ if code != http.StatusOK {
+ t.Fatalf("status = %d, want 200", code)
+ }
+ if body != "over the socket" {
+ t.Fatalf("body = %q, want socket backend body", body)
+ }
+ if *seenHost != "sock.example.com" {
+ t.Fatalf("backend saw Host %q, want sock.example.com", *seenHost)
+ }
+}
+
+func TestFirstFingerprintObservationReloads(t *testing.T) {
+ backend, _ := hostEchoBackend(t, "seen on first poll")
+ addr := backend.Listener.Addr().String()
+
+ // Snapshot BEFORE the route exists: New's startup reload sees zero routes.
+ reg := openReg(t)
+ e, err := New(reg, nil)
+ if err != nil {
+ t.Fatalf("New: %v", err)
+ }
+ srv := httptest.NewServer(e)
+ defer srv.Close()
+
+ // The route lands after the startup reload but before the watcher starts —
+ // exactly the write a baseline-only first tick would silently absorb.
+ reg2, err := registry.Open(reg.Path())
+ if err != nil {
+ t.Fatalf("open second handle: %v", err)
+ }
+ if err := reg2.Upsert(registry.Route{Host: "first.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+
+ stop := make(chan struct{})
+ defer close(stop)
+ go e.Watch(stop)
+
+ // No second write: the FIRST observed fingerprint alone must reload.
+ deadline := time.Now().Add(6 * time.Second)
+ for time.Now().Before(deadline) {
+ if code, body := getHost(t, srv.URL, "first.example.com"); code == http.StatusOK {
+ if body != "seen on first poll" {
+ t.Fatalf("body = %q, want backend body", body)
+ }
+ return
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ t.Fatalf("route never served: first fingerprint observation was baselined away instead of reloading")
+}
+
+// fakeActivator records the hibernation hook's calls.
+type fakeActivator struct {
+ served int
+ retired []map[string]bool
+}
+
+func (f *fakeActivator) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool {
+ f.served++
+ proxy.ServeHTTP(w, r) // "wake" is instant; proxy straight through
+ return true
+}
+
+func (f *fakeActivator) RetireExcept(keep map[string]bool) {
+ f.retired = append(f.retired, keep)
+}
+
+func TestHibernatingRouteGoesThroughActivator(t *testing.T) {
+ backend, _ := hostEchoBackend(t, "woken")
+ addr := backend.Listener.Addr().String()
+
+ reg := openReg(t)
+ if err := reg.Upsert(registry.Route{Host: "hib.example.com", Kind: "instance", Addr: addr, Status: "ready", Hibernate: true}); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+
+ act := &fakeActivator{}
+ e, err := New(reg, act)
+ if err != nil {
+ t.Fatalf("New: %v", err)
+ }
+ srv := httptest.NewServer(e)
+ defer srv.Close()
+
+ code, body := getHost(t, srv.URL, "hib.example.com")
+ if code != http.StatusOK || body != "woken" {
+ t.Fatalf("got %d %q, want 200 woken", code, body)
+ }
+ if act.served != 1 {
+ t.Fatalf("activator ServeHTTP called %d times, want 1", act.served)
+ }
+ // Reload reconciles the activator with the new route set.
+ if len(act.retired) == 0 {
+ t.Fatalf("RetireExcept never called on reload")
+ }
+ last := act.retired[len(act.retired)-1]
+ if !last["hib.example.com"] {
+ t.Fatalf("RetireExcept keep-set %v missing hib.example.com", last)
+ }
+}
diff --git a/internal/edge/static.go b/internal/edge/static.go
new file mode 100644
index 000000000..7a6d63340
--- /dev/null
+++ b/internal/edge/static.go
@@ -0,0 +1,20 @@
+package edge
+
+// The static serving class (routes with Kind=="static"): channel-tracked file
+// trees served at the edge. This file is a stub — Task A8 replaces it with
+// the real handler (resolves the route's release-channel pointer, extracts
+// the sha256-verified site tar into a cache dir, serves with http.FileServer
+// semantics). Until then a static route answers 404, so the route can exist
+// (and its cert can be minted) before the serving class lands.
+
+import (
+ "net/http"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+func staticHandler(rt registry.Route) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "static serving not yet implemented", http.StatusNotFound)
+ })
+}
diff --git a/main.go b/main.go
index db575cdf9..41be62bfe 100644
--- a/main.go
+++ b/main.go
@@ -7,6 +7,7 @@ import (
"log"
"os"
+ "github.com/carlosframework/platform/internal/edge"
"github.com/carlosframework/platform/internal/version"
)
@@ -16,11 +17,12 @@ func main() {
os.Exit(2)
}
cmd, args := os.Args[1], os.Args[2:]
- _ = args
switch cmd {
case "version":
fmt.Printf("carlos %s\n", version.Build)
- case "edge", "agent", "add", "remove", "instances",
+ case "edge":
+ edge.Run(args)
+ case "agent", "add", "remove", "instances",
"ship", "promote", "channels", "canary",
"bootstrap", "ops", "vet":
log.Fatalf("not implemented: %s", cmd)