eleven / ocho Public

registry: a first-class canary flag on instances — the signal, not the naming convention (#989)

An instance being a throwaway test host was only ever a hostname
convention (test-*.11ch.at), which nothing could act on — the gap that
blocked resetting a canary safely: no way to tell disposable from real
member data. Canary is now a stored Route flag, mirroring hibernate:
add-instance --canary sets it, 'lchat canary <host> --on/--off' flags an
existing instance, and it shows as a CANARY column in 'lchat instances'.

Deliberately NEVER inferred from the hostname — test.luan.chat and
test-bots.luan.chat carry the prefix but hold real data, so name-matching
would be dangerously wrong. The flag is the foundation a future scoped
reset/wipe gates on (refuse anything not flagged), and lets fleet ops and
status treat canaries distinctly. Additive migration + round-trip test.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Paul Campbell 3 months ago 13b4b64b73bcf3c879302d283770db8a8ab6ea62 parent 8f8f1ca

5 files changed, +161 −12

control.go +52 −3
control_test.go +79 −0
docs/planet-scale-guide/operations.html +2 −1
main.go +3 −0
registry.go +25 −8

diff --git a/control.go b/control.go

index fe8c353..654598c 100644

--- a/control.go

+++ b/control.go

@@ -128,6 +128,8 @@ type addInstanceOpts struct {

// no systemd unit is created — the router wakes it on first request. Needs

// the router running with LCHAT_HIBERNATE_BUCKET, or the host never answers.

Hibernate bool

+ // Canary flags the instance as a throwaway test host (registry.go: Route.Canary).

+ Canary bool

// Replication is the durability class on the activator's S3 replica

// (registry.go: stream/batched/daily; "" = stream) — set at birth so a

// provisioned space starts on its plan's rung instead of the most

@@ -205,7 +207,7 @@ func registerInstance(reg *Registry, opts addInstanceOpts) (Route, string, error

DBPath: filepath.Join(instanceDataDir, host+".db"), Status: "creating",

AppStoreURL: opts.AppStoreURL, HomeOrigin: opts.HomeOrigin, PushRelayURL: opts.PushRelayURL,

HomeRegion: opts.HomeRegion, Provider: opts.Provider, Hibernate: opts.Hibernate,

- Replication: opts.Replication}

+ Canary: opts.Canary, Replication: opts.Replication}

if err := reg.upsert(route); err != nil {

return Route{}, "", fmt.Errorf("register: %v", err)

}

@@ -362,6 +364,7 @@ func runAddInstance(args []string) {

pushRelayToken := fs.String("push-relay-token", "", "per-instance LCHAT_PUSH_RELAY_TOKEN")

hibernate := fs.Bool("hibernate", false, "register as activator-managed (no systemd unit; the router wakes it on demand — requires the router to run with LCHAT_HIBERNATE_BUCKET)")

replication := fs.String("replication", "", "replication class on the activator's S3 replica: stream (default), batched, or daily — the per-plan durability ladder; inert without --hibernate")

+ canary := fs.Bool("canary", false, "mark as a throwaway test instance (a per-session canary) — a deliberate signal, never inferred from the hostname; lets fleet ops and a future scoped reset treat it as disposable")

_ = fs.Parse(reorderFlagsFirst(fs, args))

reg, err := openRegistry(*registryPath)

@@ -376,6 +379,7 @@ func runAddInstance(args []string) {

PushRelayURL: *pushRelayURL,

PushRelayToken: *pushRelayToken,

Hibernate: *hibernate,

+ Canary: *canary,

Replication: strings.ToLower(strings.TrimSpace(*replication)),

UnitTmpl: *unitTmpl,

RouterUnit: *routerUnit,

@@ -388,6 +392,9 @@ func runAddInstance(args []string) {

if *hibernate {

fmt.Printf(" hibernating: no systemd unit — the router wakes it per request and sleeps it when idle\n")

}

+ if *canary {

+ fmt.Printf(" canary: flagged as a throwaway test instance\n")

+ }

if res.SetupErr != nil {

fmt.Printf(" (setup link unavailable: %v — mint later: lchat setup-invite %s)\n", res.SetupErr, res.Host)

} else {

@@ -603,7 +610,7 @@ func runInstances(args []string) {

fmt.Println("(no routes)")

return

}

- fmt.Printf("%-34s %-9s %-9s %-12s %-9s %-11s %s\n", "HOST", "KIND", "STATUS", "VERSION", "HIBERNATE", "REPLICATION", "SOCKET")

+ fmt.Printf("%-34s %-9s %-9s %-12s %-9s %-7s %-11s %s\n", "HOST", "KIND", "STATUS", "VERSION", "HIBERNATE", "CANARY", "REPLICATION", "SOCKET")

for _, r := range routes {

ver := r.Version

if ver == "" {

@@ -626,7 +633,11 @@ func runInstances(args []string) {

repl = "-"

}

- fmt.Printf("%-34s %-9s %-9s %-12s %-9s %-11s %s\n", r.Host, r.Kind, r.Status, ver, hib, repl, r.Socket)

+ can := "-"

+ if r.Canary {

+ can = "canary"

+ }

+ fmt.Printf("%-34s %-9s %-9s %-12s %-9s %-7s %-11s %s\n", r.Host, r.Kind, r.Status, ver, hib, can, repl, r.Socket)

if r.LastError != "" {

fmt.Printf(" last error: %s\n", r.LastError)

}

@@ -709,6 +720,44 @@ func runReplication(args []string) {

// design: a production canary can hibernate while the rest of the fleet stays

// on its units. --on requires the router to be running with

// LCHAT_HIBERNATE_BUCKET, or the host will sit unwakeable once its unit stops.

+func runCanary(args []string) {

+ fs := flag.NewFlagSet("canary", flag.ExitOnError)

+ registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")

+ on := fs.Bool("on", false, "mark the instance as a throwaway canary")

+ off := fs.Bool("off", false, "clear the canary flag (a real, keep-its-data instance)")

+ _ = fs.Parse(reorderFlagsFirst(fs, args))

+ if *on == *off {

+ die("canary: pass exactly one of --on / --off")

+ }

+ host := strings.ToLower(strings.TrimSpace(fs.Arg(0)))

+ if !validHost(host) {

+ die("canary: invalid host %q", host)

+ }

+ reg, err := openRegistry(*registryPath)

+ if err != nil {

+ die("canary: open registry: %v", err)

+ }

+ rt, err := reg.get(host)

+ if err != nil {

+ die("canary: %v", err)

+ }

+ if rt == nil {

+ die("canary: unknown host %q", host)

+ }

+ if rt.Kind != "instance" {

+ die("canary: %q is a %s route, not an instance", host, rt.Kind)

+ }

+ if err := reg.setCanary(host, *on); err != nil {

+ die("canary %s: %v", host, err)

+ }

+ if *on {

+ fmt.Printf("%s is flagged as a canary (throwaway test instance)\n", host)

+ } else {

+ fmt.Printf("%s is no longer flagged as a canary\n", host)

+ }

func runHibernate(args []string) {

fs := flag.NewFlagSet("hibernate", flag.ExitOnError)

registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")

diff --git a/control_test.go b/control_test.go

index d3f8276..9c94193 100644

--- a/control_test.go

+++ b/control_test.go

@@ -939,3 +939,82 @@ func TestHibernateOnRerunFinishesHandover(t *testing.T) {

t.Fatalf("re-run must stop the unit first, got %v", calls)

}

+func TestRegistryMigratesCanaryColumn(t *testing.T) {

+ path := filepath.Join(t.TempDir(), "reg.db")

+ // The schema as of the replication column — everything current EXCEPT canary.

+ old, err := sql.Open("sqlite", path)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if _, err := old.Exec(`

+CREATE TABLE routes (

+ host TEXT PRIMARY KEY,

+ kind TEXT NOT NULL,

+ socket TEXT NOT NULL,

+ status TEXT NOT NULL DEFAULT 'ready',

+ last_error TEXT NOT NULL DEFAULT '',

+ db_path TEXT NOT NULL DEFAULT '',

+ version TEXT NOT NULL DEFAULT '',

+ prev_version TEXT NOT NULL DEFAULT '',

+ app_store_url TEXT NOT NULL DEFAULT '',

+ home_origin TEXT NOT NULL DEFAULT '',

+ push_relay_url TEXT NOT NULL DEFAULT '',

+ push_relay_token TEXT NOT NULL DEFAULT '',

+ home_region TEXT NOT NULL DEFAULT '',

+ provider TEXT NOT NULL DEFAULT '',

+ hibernate INTEGER NOT NULL DEFAULT 0,

+ replication TEXT NOT NULL DEFAULT '',

+ next_due INTEGER NOT NULL DEFAULT 0,

+ created_at INTEGER NOT NULL

+);

+INSERT INTO routes (host, kind, socket, hibernate, created_at) VALUES ('old.example.com', 'instance', '/run/lchat/old.sock', 1, 1700000000);

+`); err != nil {

+ t.Fatal(err)

+ }

+ if err := old.Close(); err != nil {

+ t.Fatal(err)

+ }

+ reg, err := openRegistry(path)

+ if err != nil {

+ t.Fatalf("open over pre-canary registry: %v", err)

+ }

+ // Existing rows read as canary=false — a live instance is never a throwaway

+ // by default; the flag is only ever set deliberately.

+ rt, err := reg.get("old.example.com")

+ if err != nil || rt == nil {

+ t.Fatalf("get pre-existing row: %+v, %v", rt, err)

+ }

+ if rt.Canary {

+ t.Error("pre-existing row: canary=true, want the never-inferred default (false)")

+ }

+ // The flag round-trips through the narrow setter, an upsert carries it, and

+ // setCanary stays orthogonal to the rest of the route (hibernate intact).

+ if err := reg.setCanary("old.example.com", true); err != nil {

+ t.Fatalf("setCanary: %v", err)

+ }

+ if got, _ := reg.get("old.example.com"); got == nil || !got.Canary || !got.Hibernate {

+ t.Fatalf("after setCanary, route = %+v, want canary + hibernate intact", got)

+ }

+ rt, _ = reg.get("old.example.com")

+ rt.Version = "v2"

+ if err := reg.upsert(*rt); err != nil {

+ t.Fatalf("upsert rewrite: %v", err)

+ }

+ if got, _ := reg.get("old.example.com"); got == nil || !got.Canary {

+ t.Fatalf("route rewrite dropped the canary flag: %+v", got)

+ }

+ // Clearing it round-trips too, and an unknown host is a quiet no-op.

+ if err := reg.setCanary("old.example.com", false); err != nil {

+ t.Fatalf("setCanary off: %v", err)

+ }

+ if got, _ := reg.get("old.example.com"); got == nil || got.Canary {

+ t.Fatalf("setCanary off left it flagged: %+v", got)

+ }

+ if err := reg.setCanary("ghost.example.com", true); err != nil {

+ t.Fatalf("setCanary unknown host should no-op: %v", err)

+ }

diff --git a/docs/planet-scale-guide/operations.html b/docs/planet-scale-guide/operations.html

index dc71c30..2b93fe8 100644

--- a/docs/planet-scale-guide/operations.html

+++ b/docs/planet-scale-guide/operations.html

@@ -30,10 +30,11 @@ systemctl list-units 'lchat*' --failed

systemctl status lchat-router lchat-control lchat-litestream --no-pager | grep -E 'Active|since'

# 2. Does the registry agree — is every intended host ready?

-lchat instances # look for STATUS=failed rows and read their last error

+lchat instances # STATUS=failed rows need their last error read; CANARY=canary rows are throwaway test hosts, safe to reset/remove

# 3. Is the fleet current on its release channels, and whose canaries are pinned?

lchat update status # AUTO=pinned rows are someone’s live experiment — leave them be

+ # (a throwaway test host is also flagged: lchat add-instance --canary, or lchat canary <host> --on; shows as CANARY in lchat instances)

# 4. Is disk comfortable? (disk, not memory, is the binding resource on these boxes)

df -h /data

diff --git a/main.go b/main.go

index 97470c4..9aa6b93 100644

--- a/main.go

+++ b/main.go

@@ -181,6 +181,9 @@ func main() {

case "hibernate":

runHibernate(os.Args[2:])

return

+ case "canary":

+ runCanary(os.Args[2:])

+ return

case "replication":

runReplication(os.Args[2:])

return

diff --git a/registry.go b/registry.go

index 75b6f87..f7d4a4b 100644

--- a/registry.go

+++ b/registry.go

@@ -43,6 +43,14 @@ type Route struct {

// hibernate while the rest of the box is untouched. Flipped by

// `lchat hibernate <host> --on/--off` via setHibernate.

Hibernate bool

+ // Canary marks a throwaway instance (a per-session test host) — the OPPOSITE

+ // of member data. It's set deliberately (add-instance --canary, or

+ // `lchat canary <host> --on`), NEVER inferred from the hostname: test.luan.chat

+ // and test-bots.luan.chat are long-lived data-holding instances despite the

+ // prefix, so name-matching would be dangerously wrong. The signal exists so a

+ // future scoped reset/wipe can refuse anything not flagged, and so fleet ops

+ // and status can treat canaries distinctly.

+ Canary bool

// Replication is the host's durability class on the activator's S3 replica

// — the per-plan ladder ("stream"/"batched"/"daily"; "" = stream, the

// historical behavior). It sets how often litestream's replica syncs local

@@ -140,6 +148,7 @@ CREATE TABLE IF NOT EXISTS routes (

home_region TEXT NOT NULL DEFAULT '',

provider TEXT NOT NULL DEFAULT '',

hibernate INTEGER NOT NULL DEFAULT 0,

+ canary INTEGER NOT NULL DEFAULT 0,

replication TEXT NOT NULL DEFAULT '',

next_due INTEGER NOT NULL DEFAULT 0,

created_at INTEGER NOT NULL

@@ -156,6 +165,7 @@ CREATE TABLE IF NOT EXISTS routes (

`ALTER TABLE routes ADD COLUMN next_due INTEGER NOT NULL DEFAULT 0`,

`ALTER TABLE routes ADD COLUMN hibernate INTEGER NOT NULL DEFAULT 0`,

`ALTER TABLE routes ADD COLUMN replication TEXT NOT NULL DEFAULT ''`,

+ `ALTER TABLE routes ADD COLUMN canary INTEGER NOT NULL DEFAULT 0`,

} {

if _, err := r.sql.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") {

return err

@@ -169,7 +179,7 @@ func (r *Registry) list() ([]Route, error) {

rows, err := r.sql.Query(`

SELECT host, kind, socket, status, last_error, db_path, version, prev_version,

app_store_url, home_origin, push_relay_url, push_relay_token,

- home_region, provider, hibernate, replication, next_due, created_at

+ home_region, provider, hibernate, canary, replication, next_due, created_at

FROM routes ORDER BY host`)

if err != nil {

return nil, err

@@ -180,7 +190,7 @@ FROM routes ORDER BY host`)

var t Route

if err := rows.Scan(&t.Host, &t.Kind, &t.Socket, &t.Status, &t.LastError,

&t.DBPath, &t.Version, &t.PrevVersion, &t.AppStoreURL, &t.HomeOrigin,

- &t.PushRelayURL, &t.PushRelayToken, &t.HomeRegion, &t.Provider, &t.Hibernate, &t.Replication, &t.NextDue, &t.CreatedAt); err != nil {

+ &t.PushRelayURL, &t.PushRelayToken, &t.HomeRegion, &t.Provider, &t.Hibernate, &t.Canary, &t.Replication, &t.NextDue, &t.CreatedAt); err != nil {

return nil, err

}

out = append(out, t)

@@ -194,10 +204,10 @@ func (r *Registry) get(host string) (*Route, error) {

err := r.sql.QueryRow(`

SELECT host, kind, socket, status, last_error, db_path, version, prev_version,

app_store_url, home_origin, push_relay_url, push_relay_token,

- home_region, provider, hibernate, replication, next_due, created_at

+ home_region, provider, hibernate, canary, replication, next_due, created_at

FROM routes WHERE host = ?`, host).Scan(&t.Host, &t.Kind, &t.Socket, &t.Status,

&t.LastError, &t.DBPath, &t.Version, &t.PrevVersion, &t.AppStoreURL, &t.HomeOrigin,

- &t.PushRelayURL, &t.PushRelayToken, &t.HomeRegion, &t.Provider, &t.Hibernate, &t.Replication, &t.NextDue, &t.CreatedAt)

+ &t.PushRelayURL, &t.PushRelayToken, &t.HomeRegion, &t.Provider, &t.Hibernate, &t.Canary, &t.Replication, &t.NextDue, &t.CreatedAt)

if err == sql.ErrNoRows {

return nil, nil

}

@@ -230,8 +240,8 @@ func (r *Registry) upsert(t Route) error {

_, err := r.sql.Exec(`

INSERT INTO routes (host, kind, socket, status, last_error, db_path, version, prev_version,

app_store_url, home_origin, push_relay_url, push_relay_token,

- home_region, provider, hibernate, replication, next_due, created_at)

-VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)

+ home_region, provider, hibernate, canary, replication, next_due, created_at)

+VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)

ON CONFLICT(host) DO UPDATE SET

kind=excluded.kind, socket=excluded.socket, status=excluded.status,

last_error=excluded.last_error, db_path=excluded.db_path, version=excluded.version,

@@ -239,10 +249,10 @@ ON CONFLICT(host) DO UPDATE SET

app_store_url=excluded.app_store_url, home_origin=excluded.home_origin,

push_relay_url=excluded.push_relay_url, push_relay_token=excluded.push_relay_token,

home_region=excluded.home_region, provider=excluded.provider,

- hibernate=excluded.hibernate, replication=excluded.replication, next_due=excluded.next_due`,

+ hibernate=excluded.hibernate, canary=excluded.canary, replication=excluded.replication, next_due=excluded.next_due`,

t.Host, t.Kind, t.Socket, t.Status, t.LastError, t.DBPath, t.Version, t.PrevVersion,

t.AppStoreURL, t.HomeOrigin, t.PushRelayURL, t.PushRelayToken,

- t.HomeRegion, t.Provider, t.Hibernate, t.Replication, t.NextDue, t.CreatedAt)

+ t.HomeRegion, t.Provider, t.Hibernate, t.Canary, t.Replication, t.NextDue, t.CreatedAt)

return err

}

@@ -263,6 +273,13 @@ func (r *Registry) setHibernate(host string, on bool) error {

// and storing it verbatim (replicationSyncInterval fail-safes unknowns to

// stream) keeps the space's failover working through the skew, converging when

// this box updates. An unknown host is a quiet no-op.

+// setCanary flips host's throwaway-instance marker — a narrow UPDATE like

+// setHibernate, no other field touched. An unknown host is a quiet no-op.

+func (r *Registry) setCanary(host string, on bool) error {

+ _, err := r.sql.Exec(`UPDATE routes SET canary = ? WHERE host = ?`, on, host)

+ return err

func (r *Registry) setReplication(host, class string) error {

if !validReplication(class) {

return fmt.Errorf("unknown replication class %q (want %s, %s or %s)",