| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "crypto/sha256" |
| 7 | "encoding/hex" |
| 8 | "errors" |
| 9 | "os" |
| 10 | "path/filepath" |
| 11 | "strings" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/release" |
| 16 | ) |
| 17 | |
| 18 | type legFixture struct { |
| 19 | t *testing.T |
| 20 | store *release.DirStore |
| 21 | priv ed25519.PrivateKey |
| 22 | leg *systemLeg |
| 23 | dd string |
| 24 | opt string |
| 25 | now time.Time |
| 26 | } |
| 27 | |
| 28 | func newLegFixture(t *testing.T, channel, build string) *legFixture { |
| 29 | t.Helper() |
| 30 | pub, priv, err := ed25519.GenerateKey(nil) |
| 31 | if err != nil { |
| 32 | t.Fatalf("generate key: %v", err) |
| 33 | } |
| 34 | f := &legFixture{t: t, store: release.NewDirStore(t.TempDir()), priv: priv, dd: t.TempDir(), opt: t.TempDir(), |
| 35 | now: time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC)} |
| 36 | getenv := func(k string) string { |
| 37 | if k == systemChannelEnv { |
| 38 | return channel |
| 39 | } |
| 40 | return "" |
| 41 | } |
| 42 | f.leg = newSystemLeg(f.store, pub, getenv, f.dd, f.opt) |
| 43 | f.leg.build = build |
| 44 | f.leg.platform = "linux-arm64" |
| 45 | f.leg.now = func() time.Time { return f.now } |
| 46 | f.leg.jitter = func(d time.Duration) time.Duration { return d / 2 } // deterministic: half the spread |
| 47 | return f |
| 48 | } |
| 49 | |
| 50 | func (f *legFixture) promote(version string, body []byte, spread time.Duration) { |
| 51 | f.t.Helper() |
| 52 | sum := sha256.Sum256(body) |
| 53 | rel := release.SystemRelease{Version: version, Artifacts: map[string]release.SystemArtifact{ |
| 54 | "linux-arm64": {Key: release.SystemArtifactKey(version, "linux-arm64"), SHA256: hex.EncodeToString(sum[:])}}} |
| 55 | if _, err := f.store.Put(context.Background(), rel.Artifacts["linux-arm64"].Key, body, ""); err != nil { |
| 56 | f.t.Fatal(err) |
| 57 | } |
| 58 | if _, err := release.PromoteSystem(context.Background(), f.store, release.RootSigner(f.priv), release.SystemComponentRouter, f.leg.channel, rel, |
| 59 | release.SystemPromoteOptions{Spread: spread}); err != nil { |
| 60 | f.t.Fatal(err) |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | // replaceArtifact swaps the object under the signed sha. DirStore.Put is |
| 65 | // create-only with an empty if-match, so the overwrite carries the current |
| 66 | // etag — the point is a store whose bytes no longer match the pointer. |
| 67 | func (f *legFixture) replaceArtifact(version string, body []byte) { |
| 68 | f.t.Helper() |
| 69 | key := release.SystemArtifactKey(version, "linux-arm64") |
| 70 | _, etag, err := f.store.Get(context.Background(), key) |
| 71 | if err != nil { |
| 72 | f.t.Fatal(err) |
| 73 | } |
| 74 | if _, err := f.store.Put(context.Background(), key, body, etag); err != nil { |
| 75 | f.t.Fatal(err) |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | func (f *legFixture) marker() (os.FileInfo, error) { |
| 80 | return os.Stat(filepath.Join(f.dd, systemRequestMarker)) |
| 81 | } |
| 82 | |
| 83 | func (f *legFixture) request() *systemRequest { |
| 84 | var r systemRequest |
| 85 | if err := readJSONFile(systemRequestPath(f.dd), &r); err != nil { |
| 86 | return nil |
| 87 | } |
| 88 | return &r |
| 89 | } |
| 90 | |
| 91 | func TestSystemLegDoesNothingWithoutAPointer(t *testing.T) { |
| 92 | f := newLegFixture(t, "stable", "v1") |
| 93 | f.leg.observe(context.Background()) |
| 94 | f.leg.tick(context.Background(), f.now.Add(time.Hour)) |
| 95 | if _, err := f.marker(); err == nil { |
| 96 | t.Fatal("marker touched with no pointer") |
| 97 | } |
| 98 | } |
| 99 | |
| 100 | func TestSystemLegFetchesVerifiesAndRequestsAfterTheSpread(t *testing.T) { |
| 101 | f := newLegFixture(t, "stable", "v1") |
| 102 | body := []byte("ELF v2") |
| 103 | f.promote("v2", body, 10*time.Minute) |
| 104 | f.leg.observe(context.Background()) |
| 105 | // One observe with a pointer present arms the pending update — this is |
| 106 | // what fastConvergeLoop's observe-at-arming relies on. |
| 107 | if f.leg.pending == nil || f.leg.pending.version != "v2" { |
| 108 | t.Fatalf("observe armed pending = %+v, want v2", f.leg.pending) |
| 109 | } |
| 110 | if _, err := f.marker(); err == nil { |
| 111 | t.Fatal("requested before the spread elapsed") |
| 112 | } |
| 113 | f.leg.tick(context.Background(), f.now.Add(4*time.Minute)) // jitter = 5m |
| 114 | if _, err := f.marker(); err == nil { |
| 115 | t.Fatal("requested before the deadline") |
| 116 | } |
| 117 | f.leg.tick(context.Background(), f.now.Add(5*time.Minute+time.Second)) |
| 118 | if _, err := f.marker(); err != nil { |
| 119 | t.Fatal("marker not touched after the deadline") |
| 120 | } |
| 121 | got, _ := os.ReadFile(systemCachePath(f.dd, "v2")) |
| 122 | if string(got) != "ELF v2" { |
| 123 | t.Fatalf("cache = %q", got) |
| 124 | } |
| 125 | r := f.request() |
| 126 | if r == nil || r.Version != "v2" || r.NoOp || r.PromotedAt.IsZero() { |
| 127 | t.Fatalf("request = %+v", r) |
| 128 | } |
| 129 | raw, _ := os.ReadFile(systemRequestPath(f.dd)) |
| 130 | for _, never := range []string{"channel", "key", "sha", "path"} { |
| 131 | if strings.Contains(string(raw), `"`+never+`"`) { |
| 132 | t.Errorf("request carries %q — root must not take that from the sandbox", never) |
| 133 | } |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | func TestSystemLegRetouchesUntilRootAnswers(t *testing.T) { |
| 138 | t.Parallel() |
| 139 | f := newLegFixture(t, "stable", "v1") |
| 140 | f.promote("v2", []byte("x"), 0) |
| 141 | f.leg.observe(context.Background()) |
| 142 | f.leg.tick(context.Background(), f.now) |
| 143 | first, _ := f.marker() |
| 144 | f.leg.tick(context.Background(), f.now.Add(30*time.Second)) |
| 145 | same, _ := f.marker() |
| 146 | if !same.ModTime().Equal(first.ModTime()) { |
| 147 | t.Fatal("re-touched inside the minute throttle") |
| 148 | } |
| 149 | // The touch is a rewrite (PathChanged needs a change), so mtime moves. |
| 150 | time.Sleep(20 * time.Millisecond) |
| 151 | f.leg.tick(context.Background(), f.now.Add(61*time.Second)) |
| 152 | later, _ := f.marker() |
| 153 | if !later.ModTime().After(first.ModTime()) { |
| 154 | t.Fatal("not re-touched after a minute with no result") |
| 155 | } |
| 156 | // Root answers: a terminal result for this tuple stops the re-touch and |
| 157 | // the leg deletes its own cache. |
| 158 | r := f.request() |
| 159 | os.MkdirAll(systemRootDir(f.opt), 0o755) |
| 160 | writeJSONDurable(filepath.Join(systemRootDir(f.opt), "result.json"), |
| 161 | systemResult{Channel: "stable", Version: "v2", PromotedAt: r.PromotedAt, Outcome: systemOutcomeRolledBack, Reason: "gate", At: f.now}, 0o644) |
| 162 | time.Sleep(20 * time.Millisecond) |
| 163 | f.leg.tick(context.Background(), f.now.Add(3*time.Minute)) |
| 164 | final, _ := f.marker() |
| 165 | if !final.ModTime().Equal(later.ModTime()) { |
| 166 | t.Fatal("re-touched after a terminal result") |
| 167 | } |
| 168 | if _, err := os.Stat(systemCachePath(f.dd, "v2")); err == nil { |
| 169 | t.Fatal("cache kept after a terminal result") |
| 170 | } |
| 171 | // And a later observe of the SAME pointer does nothing: refuse to repeat a failure. |
| 172 | f.leg.observe(context.Background()) |
| 173 | f.leg.tick(context.Background(), f.now.Add(10*time.Minute)) |
| 174 | if f.leg.pending != nil { |
| 175 | t.Fatal("re-requested a pointer root already rolled back") |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | func TestSystemLegNoOpRequestFetchesNothing(t *testing.T) { |
| 180 | f := newLegFixture(t, "stable", "v2") |
| 181 | f.promote("v2", []byte("same"), 0) |
| 182 | f.leg.observe(context.Background()) |
| 183 | f.leg.tick(context.Background(), f.now) |
| 184 | r := f.request() |
| 185 | if r == nil || !r.NoOp { |
| 186 | t.Fatalf("request = %+v, want a no-op", r) |
| 187 | } |
| 188 | if _, err := os.Stat(systemCachePath(f.dd, "v2")); err == nil { |
| 189 | t.Fatal("a no-op must not download the artifact") |
| 190 | } |
| 191 | // Once root has seen it (seen.json advanced to this PromotedAt), nothing more. |
| 192 | os.MkdirAll(systemRootDir(f.opt), 0o755) |
| 193 | writeJSONDurable(filepath.Join(systemRootDir(f.opt), "seen.json"), systemSeen{"stable": r.PromotedAt}, 0o644) |
| 194 | f.leg.pending = nil |
| 195 | os.Remove(filepath.Join(f.dd, systemRequestMarker)) |
| 196 | f.leg.observe(context.Background()) |
| 197 | f.leg.tick(context.Background(), f.now.Add(2*time.Minute)) |
| 198 | if _, err := f.marker(); err == nil { |
| 199 | t.Fatal("requested again after root recorded the watermark") |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | func TestSystemLegIgnoresDevBuildEqualityAndOtherPlatforms(t *testing.T) { |
| 204 | f := newLegFixture(t, "stable", "dev") |
| 205 | f.promote("v2", []byte("x"), 0) |
| 206 | f.leg.platform = "linux-amd64" // no artifact for me |
| 207 | f.leg.observe(context.Background()) |
| 208 | f.leg.tick(context.Background(), f.now) |
| 209 | if _, err := f.marker(); err == nil { |
| 210 | t.Fatal("requested with no artifact for this platform") |
| 211 | } |
| 212 | } |
| 213 | |
| 214 | func TestSystemLegRefusesAnArtifactWhoseShaDoesNotMatch(t *testing.T) { |
| 215 | f := newLegFixture(t, "stable", "v1") |
| 216 | f.promote("v2", []byte("good"), 0) |
| 217 | // Swap the object under the signed sha. |
| 218 | f.replaceArtifact("v2", []byte("evil")) |
| 219 | f.leg.observe(context.Background()) |
| 220 | f.leg.tick(context.Background(), f.now) |
| 221 | if _, err := f.marker(); err == nil { |
| 222 | t.Fatal("requested an artifact whose sha does not match") |
| 223 | } |
| 224 | if _, err := os.Stat(systemCachePath(f.dd, "v2")); err == nil { |
| 225 | t.Fatal("mismatched artifact left in the cache") |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | func TestSystemLegIgnoresAPointerOlderThanTheWatermark(t *testing.T) { |
| 230 | f := newLegFixture(t, "stable", "v1") |
| 231 | f.promote("v2", []byte("x"), 0) |
| 232 | os.MkdirAll(systemRootDir(f.opt), 0o755) |
| 233 | writeJSONDurable(filepath.Join(systemRootDir(f.opt), "seen.json"), systemSeen{"stable": time.Now().Add(time.Hour)}, 0o644) |
| 234 | f.leg.observe(context.Background()) |
| 235 | f.leg.tick(context.Background(), f.now) |
| 236 | if _, err := f.marker(); err == nil { |
| 237 | t.Fatal("requested a pointer root would refuse as a replay") |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | // countFetches wraps the leg's fetch seam with a counter, leaving the real |
| 242 | // download in place, and returns a pointer to the count. |
| 243 | func (f *legFixture) countFetches() *int { |
| 244 | n := 0 |
| 245 | real := f.leg.fetchTo |
| 246 | f.leg.fetchTo = func(ctx context.Context, key, dst, sha string) error { |
| 247 | n++ |
| 248 | return real(ctx, key, dst, sha) |
| 249 | } |
| 250 | return &n |
| 251 | } |
| 252 | |
| 253 | func TestSystemLegBacksOffAGrowingWindowAfterAFailedFetch(t *testing.T) { |
| 254 | f := newLegFixture(t, "stable", "v1") |
| 255 | f.promote("v2", []byte("good"), 0) |
| 256 | fetches := 0 |
| 257 | f.leg.fetchTo = func(ctx context.Context, key, dst, sha string) error { |
| 258 | fetches++ |
| 259 | return errors.New("store unreachable") |
| 260 | } |
| 261 | ctx := context.Background() |
| 262 | f.leg.observe(ctx) |
| 263 | f.leg.tick(ctx, f.now) // first attempt: hold off 2s |
| 264 | if fetches != 1 { |
| 265 | t.Fatalf("first tick fetched %d times, want 1", fetches) |
| 266 | } |
| 267 | f.leg.tick(ctx, f.now.Add(time.Second)) |
| 268 | if fetches != 1 { |
| 269 | t.Fatal("re-downloaded inside the first backoff window") |
| 270 | } |
| 271 | f.leg.tick(ctx, f.now.Add(3*time.Second)) // second attempt: hold off 4s |
| 272 | if fetches != 2 { |
| 273 | t.Fatalf("after the 2s window fetched %d times, want 2", fetches) |
| 274 | } |
| 275 | f.leg.tick(ctx, f.now.Add(5*time.Second)) |
| 276 | if fetches != 2 { |
| 277 | t.Fatal("backoff did not grow: re-downloaded 2s after the second failure") |
| 278 | } |
| 279 | f.leg.tick(ctx, f.now.Add(8*time.Second)) // third attempt: hold off 8s |
| 280 | if fetches != 3 { |
| 281 | t.Fatalf("after the 4s window fetched %d times, want 3", fetches) |
| 282 | } |
| 283 | if f.leg.backoff != 8*time.Second { |
| 284 | t.Fatalf("ladder = %s after three failures, want 8s", f.leg.backoff) |
| 285 | } |
| 286 | if _, err := f.marker(); err == nil { |
| 287 | t.Fatal("asked root to apply an artifact that never downloaded") |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | func TestSystemLegReusesAVerifiedCacheOnARetry(t *testing.T) { |
| 292 | f := newLegFixture(t, "stable", "v1") |
| 293 | f.promote("v2", []byte("ELF v2"), 0) |
| 294 | fetches := f.countFetches() |
| 295 | ctx := context.Background() |
| 296 | f.leg.observe(ctx) |
| 297 | f.leg.tick(ctx, f.now) |
| 298 | if *fetches != 1 { |
| 299 | t.Fatalf("first tick fetched %d times, want 1", *fetches) |
| 300 | } |
| 301 | // Root defers: it could not reach the store to verify the pointer. |
| 302 | r := f.request() |
| 303 | os.MkdirAll(systemRootDir(f.opt), 0o755) |
| 304 | writeJSONDurable(filepath.Join(systemRootDir(f.opt), "result.json"), |
| 305 | systemResult{Channel: "stable", Version: "v2", PromotedAt: r.PromotedAt, Outcome: systemOutcomeDeferred, |
| 306 | Reason: "store", At: f.now.Add(500 * time.Millisecond)}, 0o644) |
| 307 | f.leg.tick(ctx, f.now.Add(time.Second)) // sees the deferral, re-deadlines |
| 308 | f.leg.tick(ctx, f.now.Add(4*time.Second)) // retries |
| 309 | if *fetches != 1 { |
| 310 | t.Fatalf("retry re-downloaded an artifact the cache already holds (%d fetches)", *fetches) |
| 311 | } |
| 312 | if got, _ := os.ReadFile(systemCachePath(f.dd, "v2")); string(got) != "ELF v2" { |
| 313 | t.Fatalf("cache = %q after the retry", got) |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | func TestSystemLegSaysNotForThisBoxOnce(t *testing.T) { |
| 318 | f := newLegFixture(t, "stable", "v1") |
| 319 | f.promote("v2", []byte("x"), 0) |
| 320 | f.leg.platform = "linux-amd64" // no artifact for me |
| 321 | lines := 0 |
| 322 | f.leg.logf = func(format string, args ...any) { lines++ } |
| 323 | for i := 0; i < 3; i++ { |
| 324 | f.leg.observe(context.Background()) |
| 325 | } |
| 326 | if lines != 1 { |
| 327 | t.Fatalf("three observes of the same pointer logged %d lines, want 1", lines) |
| 328 | } |
| 329 | } |
| 330 | |
| 331 | func TestSystemLegPrunesTheCacheOfSupersededVersions(t *testing.T) { |
| 332 | f := newLegFixture(t, "stable", "v1") |
| 333 | f.promote("v2", []byte("two"), 0) |
| 334 | ctx := context.Background() |
| 335 | f.leg.observe(ctx) |
| 336 | f.leg.tick(ctx, f.now) |
| 337 | if _, err := os.Stat(systemCachePath(f.dd, "v2")); err != nil { |
| 338 | t.Fatalf("v2 not cached: %v", err) |
| 339 | } |
| 340 | // v3 lands before root ever answered for v2. |
| 341 | f.promote("v3", []byte("three"), 0) |
| 342 | f.leg.observe(ctx) |
| 343 | if _, err := os.Stat(systemCachePath(f.dd, "v2")); err == nil { |
| 344 | t.Fatal("v2's binary kept after v3 superseded it") |
| 345 | } |
| 346 | if f.request() == nil { |
| 347 | t.Fatal("the prune took the request with it") |
| 348 | } |
| 349 | } |
| 350 | |
| 351 | func TestFastTickRunsTheSystemLeg(t *testing.T) { |
| 352 | fx := newFastFixture(t) |
| 353 | d := fx.deps(nil) |
| 354 | observes := 0 |
| 355 | d.systemLeg = &systemLeg{observeHook: func() { observes++ }} |
| 356 | ctx := context.Background() |
| 357 | fastTick(ctx, d, time.Now()) // boot seed |
| 358 | if observes != 0 { |
| 359 | t.Fatalf("boot seed observed %d times", observes) |
| 360 | } |
| 361 | release.BumpSerial(ctx, fx.store, time.Now()) |
| 362 | fastTick(ctx, d, time.Now().Add(10*time.Second)) |
| 363 | if observes != 1 { |
| 364 | t.Fatalf("one serial move → %d observes", observes) |
| 365 | } |
| 366 | } |
| 367 | |
amadan