| 1 | package main |
| 2 | |
| 3 | // The sibling services — connect broker, push relay, home server — run from |
| 4 | // loose /opt/lchat/<name> binaries that root used to scp onto the box, a path |
| 5 | // the operator-allowlist hardening removed (docs/instance-router.md § Box |
| 6 | // access). They now converge from the SAME signed manifest entry the control |
| 7 | // binary follows (controlSelfUpdate): publish uploads lchat-<name>-linux-<arch> |
| 8 | // beside the instance binaries and pins their sha256s INSIDE the signed |
| 9 | // payload, and the already-root hourly pass swaps any binary whose on-disk |
| 10 | // sha doesn't match — same signature, same bake window, same rollback shape. |
| 11 | // |
| 12 | // Convergence is sha-driven, not version-driven: the services carry no build |
| 13 | // stamp, and comparing hashes also heals a tampered or half-written binary. |
| 14 | // A box that doesn't run a service simply has no /opt/lchat/<name> file and |
| 15 | // is skipped — nothing is ever installed onto a box that didn't opt in. |
| 16 | |
| 17 | import ( |
| 18 | "crypto/sha256" |
| 19 | "encoding/hex" |
| 20 | "fmt" |
| 21 | "io" |
| 22 | "log" |
| 23 | "os" |
| 24 | "path/filepath" |
| 25 | "strings" |
| 26 | "time" |
| 27 | ) |
| 28 | |
| 29 | // serviceBinaries maps a manifest `services` key to the loose binary path |
| 30 | // component under optDir and the unit that execs it. Order is cosmetic. |
| 31 | // lchat-links is deliberately absent: it's a subcommand of the control binary, |
| 32 | // so it rides controlSelfUpdate. |
| 33 | var serviceBinaries = []struct { |
| 34 | Name string // manifest key AND /opt/lchat/<Name> |
| 35 | Unit string |
| 36 | }{ |
| 37 | {"connect", "lchat-connect"}, |
| 38 | {"push", "lchat-push"}, |
| 39 | {"home", "lchat-home"}, |
| 40 | } |
| 41 | |
| 42 | // serviceHealthSettle is how long a bounced unit gets to crash before we call |
| 43 | // the swap good — Restart=always units flap fast, so a binary that can't boot |
| 44 | // shows as inactive/activating within a beat. A var so tests shrink it. |
| 45 | var serviceHealthSettle = 3 * time.Second |
| 46 | |
| 47 | func serviceFailedPath(name string) string { |
| 48 | return filepath.Join(optDir, ".service-failed-"+name) |
| 49 | } |
| 50 | |
| 51 | func readServiceFailed(name string) string { |
| 52 | b, err := os.ReadFile(serviceFailedPath(name)) |
| 53 | if err != nil { |
| 54 | return "" |
| 55 | } |
| 56 | return strings.TrimSpace(string(b)) |
| 57 | } |
| 58 | |
| 59 | // fileSHA256 hex-hashes a file; os.IsNotExist(err) is the caller's "this box |
| 60 | // doesn't run the service" signal. |
| 61 | func fileSHA256(path string) (string, error) { |
| 62 | f, err := os.Open(path) |
| 63 | if err != nil { |
| 64 | return "", err |
| 65 | } |
| 66 | defer f.Close() |
| 67 | h := sha256.New() |
| 68 | if _, err := io.Copy(h, f); err != nil { |
| 69 | return "", err |
| 70 | } |
| 71 | return hex.EncodeToString(h.Sum(nil)), nil |
| 72 | } |
| 73 | |
| 74 | // serviceBinariesUpdate converges the sibling-service binaries onto the |
| 75 | // control channel's head. Runs from updatePass AFTER controlSelfUpdate — on |
| 76 | // the pass that swaps the control binary this (still-old) process runs the |
| 77 | // converge with old code, which is fine: the logic is data-driven from the |
| 78 | // manifest, and the next pass runs it from the new build anyway. Like the |
| 79 | // control swap, failures log and never fail the pass — the instances above |
| 80 | // already converged. |
| 81 | func serviceBinariesUpdate(m *releaseManifest, bake *firstSeenState, poisoned map[string]bool, dryRun, ignoreBake bool) { |
| 82 | channel := envOr("LCHAT_CONTROL_CHANNEL", "stable") |
| 83 | if channel == "off" { |
| 84 | return |
| 85 | } |
| 86 | e, ok := m.Channels[channel] |
| 87 | if !ok || len(e.Services) == 0 { |
| 88 | // Pre-feature releases (and promote's rebuilt-from-assets entries) |
| 89 | // carry no services block — nothing to converge on. |
| 90 | return |
| 91 | } |
| 92 | if poisoned[e.Version] { |
| 93 | fmt.Printf("update: skipping service binaries for %s — it failed a health check on an instance this pass\n", e.Version) |
| 94 | return |
| 95 | } |
| 96 | if hold := bakeHold(bake, channel, e.Version, ignoreBake, time.Now()); hold != "" { |
| 97 | fmt.Printf("update: service binaries -> %s: %s\n", e.Version, hold) |
| 98 | return |
| 99 | } |
| 100 | for _, svc := range serviceBinaries { |
| 101 | sb, ok := e.Services[svc.Name] |
| 102 | if !ok { |
| 103 | continue |
| 104 | } |
| 105 | path := filepath.Join(optDir, svc.Name) |
| 106 | cur, err := fileSHA256(path) |
| 107 | if err != nil { |
| 108 | // No file = this box doesn't run the service; anything else is |
| 109 | // worth a line but not a retry loop. |
| 110 | if !os.IsNotExist(err) { |
| 111 | fmt.Printf("update: service %s: %v\n", svc.Name, err) |
| 112 | } |
| 113 | continue |
| 114 | } |
| 115 | url, want, ok := sb.forArch(updaterArch) |
| 116 | if !ok { |
| 117 | continue // no build for this arch — same skip rule as instances |
| 118 | } |
| 119 | if cur == want { |
| 120 | continue |
| 121 | } |
| 122 | if failed := readServiceFailed(svc.Name); failed != "" { |
| 123 | if failed == e.Version { |
| 124 | fmt.Printf("update: service %s build %s is quarantined (failed its restart; pin or patch the channel to move)\n", svc.Name, e.Version) |
| 125 | continue |
| 126 | } |
| 127 | os.Remove(serviceFailedPath(svc.Name)) // channel moved on — retry |
| 128 | } |
| 129 | if dryRun { |
| 130 | fmt.Printf("update: service %s would update to %s (%s; dry run)\n", svc.Name, e.Version, channel) |
| 131 | continue |
| 132 | } |
| 133 | if err := swapServiceBinary(svc.Name, svc.Unit, path, url, want, e.Version); err != nil { |
| 134 | fmt.Printf("update: service %s: %v\n", svc.Name, err) |
| 135 | } |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | // swapServiceBinary downloads (sha-gated, mirror fallback), atomically swaps |
| 140 | // with a .prev kept beside it, and — only when the unit is actually running — |
| 141 | // bounces it and confirms it stays up, rolling back and quarantining on |
| 142 | // failure. An inactive unit gets the new file silently: the box staged the |
| 143 | // binary but hasn't enabled the service, so there is nothing to health-check. |
| 144 | func swapServiceBinary(name, unit, path, url, wantSha, ver string) error { |
| 145 | tmp, err := downloadBinaryToTemp(url, wantSha, ver) |
| 146 | if err != nil { |
| 147 | if releaseMirrorURL == "" { |
| 148 | return err |
| 149 | } |
| 150 | murl := mirrorBinaryURL(releaseMirrorURL, url) |
| 151 | log.Printf("update: service %s primary download failed (%v) — trying mirror %s", name, err, murl) |
| 152 | if tmp, err = downloadBinaryToTemp(murl, wantSha, ver); err != nil { |
| 153 | return fmt.Errorf("download: primary and mirror failed (%v)", err) |
| 154 | } |
| 155 | } |
| 156 | defer os.Remove(tmp) |
| 157 | if err := os.Chmod(tmp, 0o755); err != nil { |
| 158 | return fmt.Errorf("chmod: %v", err) |
| 159 | } |
| 160 | prev := path + ".prev" |
| 161 | if err := copyFileAtomic(path, prev, 0o755); err != nil { |
| 162 | return fmt.Errorf("keep rollback copy: %v", err) |
| 163 | } |
| 164 | // downloadBinaryToTemp lands in incomingDir() — same filesystem, atomic. |
| 165 | if err := os.Rename(tmp, path); err != nil { |
| 166 | return fmt.Errorf("install: %v", err) |
| 167 | } |
| 168 | if !isActive(unit) { |
| 169 | fmt.Printf("update: service %s -> %s (unit %s inactive — applies when it starts; rollback at %s)\n", name, ver, unit, prev) |
| 170 | return nil |
| 171 | } |
| 172 | if err := systemctl("try-restart", unit); err != nil { |
| 173 | fmt.Printf("update: service %s restart: %v (new build applies on its next start)\n", name, err) |
| 174 | return nil |
| 175 | } |
| 176 | time.Sleep(serviceHealthSettle) |
| 177 | if isActive(unit) { |
| 178 | os.Remove(serviceFailedPath(name)) |
| 179 | fmt.Printf("update: service %s -> %s (%s healthy; rollback at %s)\n", name, ver, unit, prev) |
| 180 | return nil |
| 181 | } |
| 182 | // The bounced unit didn't stay up: restore the known-good binary, bounce it |
| 183 | // back, and quarantine the head so the hourly pass doesn't flap forever. |
| 184 | log.Printf("update: service %s -> %s is UNHEALTHY — rolling back", name, ver) |
| 185 | if rberr := os.Rename(prev, path); rberr != nil { |
| 186 | return fmt.Errorf("SERVICE %s BRICKED: new build unhealthy AND restoring %s failed: %v", name, prev, rberr) |
| 187 | } |
| 188 | if err := systemctl("try-restart", unit); err != nil { |
| 189 | log.Printf("update: service %s rollback restart: %v", name, err) |
| 190 | } |
| 191 | if werr := os.WriteFile(serviceFailedPath(name), []byte(ver+"\n"), 0o644); werr != nil { |
| 192 | log.Printf("update: service %s: persist quarantine: %v", name, werr) |
| 193 | } |
| 194 | return fmt.Errorf("build %s failed its restart — rolled back and quarantined", ver) |
| 195 | } |
| 196 | |
amadan