| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "crypto/sha256" |
| 5 | "encoding/hex" |
| 6 | "net/http" |
| 7 | "net/http/httptest" |
| 8 | "os" |
| 9 | "path/filepath" |
| 10 | "sync/atomic" |
| 11 | "testing" |
| 12 | "time" |
| 13 | ) |
| 14 | |
| 15 | // serviceTestEnv points the service-converge seams at temp doubles: a temp |
| 16 | // optDir holding (or not holding) the loose service binary, a recording |
| 17 | // systemctl, a scriptable isActive, amd64 entries regardless of the dev |
| 18 | // machine's arch, and a zero settle so unhealthy paths fail in a beat. |
| 19 | func serviceTestEnv(t *testing.T) (calls *[]string, setActive func(func(string) bool)) { |
| 20 | t.Helper() |
| 21 | oldOpt, oldSysctl, oldActive := optDir, systemctl, isActive |
| 22 | oldArch, oldSettle := updaterArch, serviceHealthSettle |
| 23 | t.Cleanup(func() { |
| 24 | optDir, systemctl, isActive = oldOpt, oldSysctl, oldActive |
| 25 | updaterArch, serviceHealthSettle = oldArch, oldSettle |
| 26 | }) |
| 27 | optDir = t.TempDir() |
| 28 | updaterArch = "amd64" |
| 29 | serviceHealthSettle = 0 |
| 30 | t.Setenv("LCHAT_CONTROL_CHANNEL", "edge") |
| 31 | c := []string{} |
| 32 | calls = &c |
| 33 | systemctl = func(args ...string) error { |
| 34 | *calls = append(*calls, joinArgs(args)) |
| 35 | return nil |
| 36 | } |
| 37 | setActive = func(f func(string) bool) { isActive = f } |
| 38 | setActive(func(string) bool { return true }) |
| 39 | return calls, setActive |
| 40 | } |
| 41 | |
| 42 | func joinArgs(args []string) string { |
| 43 | out := "" |
| 44 | for i, a := range args { |
| 45 | if i > 0 { |
| 46 | out += " " |
| 47 | } |
| 48 | out += a |
| 49 | } |
| 50 | return out |
| 51 | } |
| 52 | |
| 53 | func shaOf(b []byte) string { |
| 54 | h := sha256.Sum256(b) |
| 55 | return hex.EncodeToString(h[:]) |
| 56 | } |
| 57 | |
| 58 | // serviceManifest builds a one-channel manifest whose edge head carries one |
| 59 | // "connect" service binary served by the given URL, plus a bake state that |
| 60 | // has already seen the head (edge bakes for 0, so recorded = eligible). |
| 61 | func serviceManifest(url string, bin []byte) (*releaseManifest, *firstSeenState) { |
| 62 | m := &releaseManifest{ |
| 63 | Schema: 1, |
| 64 | GeneratedAt: time.Now(), |
| 65 | Channels: map[string]releaseEntry{ |
| 66 | "edge": { |
| 67 | Version: "svcnew1", |
| 68 | URL: url, // instance binary URL — unused by the service pass |
| 69 | SHA256: shaOf([]byte("unused-instance-binary")), |
| 70 | Services: map[string]serviceBuild{ |
| 71 | "connect": {URL: url, SHA256: shaOf(bin)}, |
| 72 | }, |
| 73 | }, |
| 74 | }, |
| 75 | } |
| 76 | bake := &firstSeenState{Channels: map[string]map[string]time.Time{ |
| 77 | "edge": {"svcnew1": time.Now().Add(-time.Hour)}, |
| 78 | }} |
| 79 | return m, bake |
| 80 | } |
| 81 | |
| 82 | func TestServiceUpdateSwapsAndRestarts(t *testing.T) { |
| 83 | calls, _ := serviceTestEnv(t) |
| 84 | newBin := []byte("NEW-CONNECT-BINARY") |
| 85 | var hits atomic.Int32 |
| 86 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 87 | hits.Add(1) |
| 88 | w.Write(newBin) |
| 89 | })) |
| 90 | defer ts.Close() |
| 91 | |
| 92 | path := filepath.Join(optDir, "connect") |
| 93 | if err := os.WriteFile(path, []byte("OLD-CONNECT-BINARY"), 0o755); err != nil { |
| 94 | t.Fatal(err) |
| 95 | } |
| 96 | m, bake := serviceManifest(ts.URL+"/lchat-connect-linux-amd64", newBin) |
| 97 | serviceBinariesUpdate(m, bake, nil, false, false) |
| 98 | |
| 99 | got, _ := os.ReadFile(path) |
| 100 | if string(got) != string(newBin) { |
| 101 | t.Fatalf("binary not swapped: %q", got) |
| 102 | } |
| 103 | prev, err := os.ReadFile(path + ".prev") |
| 104 | if err != nil || string(prev) != "OLD-CONNECT-BINARY" { |
| 105 | t.Fatalf("rollback copy wrong: %q err %v", prev, err) |
| 106 | } |
| 107 | if len(*calls) != 1 || (*calls)[0] != "try-restart lchat-connect" { |
| 108 | t.Fatalf("systemctl calls: %v", *calls) |
| 109 | } |
| 110 | if _, err := os.Stat(serviceFailedPath("connect")); !os.IsNotExist(err) { |
| 111 | t.Fatal("healthy swap must not quarantine") |
| 112 | } |
| 113 | if hits.Load() == 0 { |
| 114 | t.Fatal("nothing downloaded") |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | func TestServiceUpdateSkipsAbsentBinary(t *testing.T) { |
| 119 | calls, _ := serviceTestEnv(t) |
| 120 | var hits atomic.Int32 |
| 121 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 122 | hits.Add(1) |
| 123 | })) |
| 124 | defer ts.Close() |
| 125 | m, bake := serviceManifest(ts.URL+"/x", []byte("NEW")) |
| 126 | serviceBinariesUpdate(m, bake, nil, false, false) |
| 127 | if hits.Load() != 0 || len(*calls) != 0 { |
| 128 | t.Fatalf("a box without the service must be untouched (hits %d, calls %v)", hits.Load(), *calls) |
| 129 | } |
| 130 | } |
| 131 | |
| 132 | func TestServiceUpdateNoopWhenCurrent(t *testing.T) { |
| 133 | calls, _ := serviceTestEnv(t) |
| 134 | cur := []byte("CURRENT-BINARY") |
| 135 | m, bake := serviceManifest("https://example.invalid/x", cur) |
| 136 | path := filepath.Join(optDir, "connect") |
| 137 | if err := os.WriteFile(path, cur, 0o755); err != nil { |
| 138 | t.Fatal(err) |
| 139 | } |
| 140 | serviceBinariesUpdate(m, bake, nil, false, false) |
| 141 | if len(*calls) != 0 { |
| 142 | t.Fatalf("sha match must be a no-op, got calls %v", *calls) |
| 143 | } |
| 144 | if _, err := os.Stat(path + ".prev"); !os.IsNotExist(err) { |
| 145 | t.Fatal("no-op must not write a rollback copy") |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | func TestServiceUpdateRollsBackUnhealthyAndQuarantines(t *testing.T) { |
| 150 | calls, setActive := serviceTestEnv(t) |
| 151 | newBin := []byte("BAD-NEW-BINARY") |
| 152 | var hits atomic.Int32 |
| 153 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 154 | hits.Add(1) |
| 155 | w.Write(newBin) |
| 156 | })) |
| 157 | defer ts.Close() |
| 158 | // Active before the swap (so the bounce happens), dead ever after. |
| 159 | var checks atomic.Int32 |
| 160 | setActive(func(string) bool { return checks.Add(1) == 1 }) |
| 161 | |
| 162 | path := filepath.Join(optDir, "connect") |
| 163 | if err := os.WriteFile(path, []byte("GOOD-OLD-BINARY"), 0o755); err != nil { |
| 164 | t.Fatal(err) |
| 165 | } |
| 166 | m, bake := serviceManifest(ts.URL+"/x", newBin) |
| 167 | serviceBinariesUpdate(m, bake, nil, false, false) |
| 168 | |
| 169 | got, _ := os.ReadFile(path) |
| 170 | if string(got) != "GOOD-OLD-BINARY" { |
| 171 | t.Fatalf("unhealthy build must roll back, file is %q", got) |
| 172 | } |
| 173 | if readServiceFailed("connect") != "svcnew1" { |
| 174 | t.Fatalf("expected quarantine of svcnew1, got %q", readServiceFailed("connect")) |
| 175 | } |
| 176 | // try-restart onto the bad build + try-restart back = 2 calls. |
| 177 | if len(*calls) != 2 { |
| 178 | t.Fatalf("systemctl calls: %v", *calls) |
| 179 | } |
| 180 | |
| 181 | // A second pass at the same head must short-circuit on the quarantine. |
| 182 | before := hits.Load() |
| 183 | serviceBinariesUpdate(m, bake, nil, false, false) |
| 184 | if hits.Load() != before { |
| 185 | t.Fatal("quarantined head was downloaded again") |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | func TestServiceUpdateInactiveUnitSwapsQuietly(t *testing.T) { |
| 190 | calls, setActive := serviceTestEnv(t) |
| 191 | setActive(func(string) bool { return false }) |
| 192 | newBin := []byte("NEW-BINARY") |
| 193 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 194 | w.Write(newBin) |
| 195 | })) |
| 196 | defer ts.Close() |
| 197 | path := filepath.Join(optDir, "connect") |
| 198 | if err := os.WriteFile(path, []byte("OLD"), 0o755); err != nil { |
| 199 | t.Fatal(err) |
| 200 | } |
| 201 | m, bake := serviceManifest(ts.URL+"/x", newBin) |
| 202 | serviceBinariesUpdate(m, bake, nil, false, false) |
| 203 | got, _ := os.ReadFile(path) |
| 204 | if string(got) != string(newBin) { |
| 205 | t.Fatalf("inactive unit still gets the file, got %q", got) |
| 206 | } |
| 207 | if len(*calls) != 0 { |
| 208 | t.Fatalf("inactive unit must not be restarted: %v", *calls) |
| 209 | } |
| 210 | } |
| 211 | |
| 212 | func TestServiceUpdateFailsClosedOnUnbakedHead(t *testing.T) { |
| 213 | calls, _ := serviceTestEnv(t) |
| 214 | var hits atomic.Int32 |
| 215 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 216 | hits.Add(1) |
| 217 | })) |
| 218 | defer ts.Close() |
| 219 | path := filepath.Join(optDir, "connect") |
| 220 | if err := os.WriteFile(path, []byte("OLD"), 0o755); err != nil { |
| 221 | t.Fatal(err) |
| 222 | } |
| 223 | m, _ := serviceManifest(ts.URL+"/x", []byte("NEW")) |
| 224 | // A bake clock that has never seen the head: fail closed, nothing moves. |
| 225 | empty := &firstSeenState{Channels: map[string]map[string]time.Time{}} |
| 226 | serviceBinariesUpdate(m, empty, nil, false, false) |
| 227 | if hits.Load() != 0 || len(*calls) != 0 { |
| 228 | t.Fatalf("unbaked head must hold (hits %d, calls %v)", hits.Load(), *calls) |
| 229 | } |
| 230 | } |
| 231 | |
| 232 | func TestServiceUpdateIgnoresEntryWithoutServices(t *testing.T) { |
| 233 | calls, _ := serviceTestEnv(t) |
| 234 | path := filepath.Join(optDir, "connect") |
| 235 | if err := os.WriteFile(path, []byte("OLD"), 0o755); err != nil { |
| 236 | t.Fatal(err) |
| 237 | } |
| 238 | m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{ |
| 239 | "edge": {Version: "oldver12", URL: "https://example.invalid/x", SHA256: shaOf([]byte("y"))}, |
| 240 | }} |
| 241 | bake := &firstSeenState{Channels: map[string]map[string]time.Time{"edge": {"oldver12": time.Now().Add(-time.Hour)}}} |
| 242 | serviceBinariesUpdate(m, bake, nil, false, false) |
| 243 | if len(*calls) != 0 { |
| 244 | t.Fatalf("pre-feature manifest must be a no-op, got %v", *calls) |
| 245 | } |
| 246 | } |
| 247 | |
amadan