| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "os" |
| 9 | "path/filepath" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/moves" |
| 14 | "github.com/carlosframework/platform/internal/registry" |
| 15 | "github.com/carlosframework/platform/internal/release" |
| 16 | ) |
| 17 | |
| 18 | // putFloor publishes the operator's floor object in s. |
| 19 | func putFloor(t *testing.T, s release.Store, minRelease string) { |
| 20 | t.Helper() |
| 21 | body, _ := json.Marshal(moves.Floor{Min: moves.ProtocolVersion, MinRelease: minRelease}) |
| 22 | if _, err := s.Put(context.Background(), moves.FloorKey, body, ""); err != nil { |
| 23 | t.Fatal(err) |
| 24 | } |
| 25 | } |
| 26 | |
| 27 | // armLocalFloor writes the box's two local floor files, as an apply would. |
| 28 | func armLocalFloor(t *testing.T, dataDir, minRelease string) { |
| 29 | t.Helper() |
| 30 | if err := writeLocalFloor(dataDir, moves.Floor{Min: moves.ProtocolVersion, MinRelease: minRelease}); err != nil { |
| 31 | t.Fatal(err) |
| 32 | } |
| 33 | } |
| 34 | |
| 35 | // The judge's held-row inputs, shaped like registry.WithHoldsFrozen. |
| 36 | func noHolds(fn func([]registry.Route) error) error { return fn(nil) } |
| 37 | |
| 38 | func oneHold(fn func([]registry.Route) error) error { |
| 39 | return fn([]registry.Route{{Host: "app.example.com", Status: "moving", HoldOp: "op1"}}) |
| 40 | } |
| 41 | |
| 42 | func unreadableHolds(func([]registry.Route) error) error { return errors.New("locked") } |
| 43 | |
| 44 | func TestLocalFloorFilesRoundTrip(t *testing.T) { |
| 45 | dd := t.TempDir() |
| 46 | f, err := readLocalFloor(dd) |
| 47 | if err != nil || f != nil { |
| 48 | t.Fatalf("no files read as %+v, %v", f, err) |
| 49 | } |
| 50 | armLocalFloor(t, dd, "v0.30.0") |
| 51 | f, err = readLocalFloor(dd) |
| 52 | if err != nil || f == nil || f.MinRelease != "v0.30.0" || f.Min != moves.ProtocolVersion { |
| 53 | t.Fatalf("armed floor read as %+v, %v", f, err) |
| 54 | } |
| 55 | line, err := os.ReadFile(filepath.Join(dd, moves.FloorMinReleaseFile)) |
| 56 | if err != nil || string(line) != "v0.30.0\n" { |
| 57 | t.Fatalf("floor.min-release = %q, %v", line, err) |
| 58 | } |
| 59 | // A floor whose min_release is not a release is never written: the |
| 60 | // units would compare garbage, and the door could never match it. |
| 61 | if err := writeLocalFloor(dd, moves.Floor{Min: 1, MinRelease: "v0.31.0\nv0.1.0"}); err == nil { |
| 62 | t.Fatal("a two-line min_release was written") |
| 63 | } |
| 64 | if f, _ := readLocalFloor(dd); f == nil || f.MinRelease != "v0.30.0" { |
| 65 | t.Fatalf("a refused write disturbed the armed floor: %+v", f) |
| 66 | } |
| 67 | } |
| 68 | |
| 69 | // The local floor lives in a directory the sandboxed agent owns, and root |
| 70 | // reads and writes it. Neither may follow a symlink planted there. |
| 71 | func TestLocalFloorRefusesASymlink(t *testing.T) { |
| 72 | dd, elsewhere := t.TempDir(), t.TempDir() |
| 73 | if err := os.Symlink(elsewhere, filepath.Join(dd, ".moves")); err != nil { |
| 74 | t.Fatal(err) |
| 75 | } |
| 76 | if err := writeLocalFloor(dd, moves.Floor{Min: 1, MinRelease: "v0.30.0"}); err == nil { |
| 77 | t.Fatal("wrote through a symlinked .moves") |
| 78 | } |
| 79 | if entries, _ := os.ReadDir(elsewhere); len(entries) != 0 { |
| 80 | t.Fatalf("the symlink's target was written: %v", entries) |
| 81 | } |
| 82 | os.WriteFile(filepath.Join(elsewhere, "floor.json"), []byte(`{"min":1,"min_release":"v0.1.0"}`), 0o644) |
| 83 | if f, err := readLocalFloor(dd); err == nil { |
| 84 | t.Fatalf("read through a symlinked .moves: %+v", f) |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | func TestSyncLocalFloorKeepsWhatIsArmed(t *testing.T) { |
| 89 | ctx := context.Background() |
| 90 | |
| 91 | // Absent everywhere: no floor, no files. |
| 92 | dd, s := t.TempDir(), release.NewDirStore(t.TempDir()) |
| 93 | if f, err := syncLocalFloor(ctx, s, dd); err != nil || f != nil { |
| 94 | t.Fatalf("absent everywhere = %+v, %v", f, err) |
| 95 | } |
| 96 | if _, err := os.Stat(filepath.Join(dd, ".moves")); !errors.Is(err, os.ErrNotExist) { |
| 97 | t.Fatal("an unarmed deployment grew a .moves directory") |
| 98 | } |
| 99 | |
| 100 | // Published: both local files written. |
| 101 | putFloor(t, s, "v0.30.0") |
| 102 | if f, err := syncLocalFloor(ctx, s, dd); err != nil || f == nil || f.MinRelease != "v0.30.0" { |
| 103 | t.Fatalf("published = %+v, %v", f, err) |
| 104 | } |
| 105 | if got, _ := readLocalFloor(dd); got == nil || got.MinRelease != "v0.30.0" { |
| 106 | t.Fatalf("local floor = %+v", got) |
| 107 | } |
| 108 | |
| 109 | // The store unreadable: the local floor stands, and the error is said. |
| 110 | if f, err := syncLocalFloor(ctx, unreadableStore{s}, dd); err == nil || f == nil || f.MinRelease != "v0.30.0" { |
| 111 | t.Fatalf("store error = %+v, %v", f, err) |
| 112 | } |
| 113 | |
| 114 | // The object gone (it is create-only, so an operator removed it): once |
| 115 | // armed, the floor stays armed. |
| 116 | if err := s.Delete(ctx, moves.FloorKey); err != nil { |
| 117 | t.Fatal(err) |
| 118 | } |
| 119 | if f, err := syncLocalFloor(ctx, s, dd); err != nil || f == nil || f.MinRelease != "v0.30.0" { |
| 120 | t.Fatalf("object removed = %+v, %v", f, err) |
| 121 | } |
| 122 | |
| 123 | // Raised: the local files follow. |
| 124 | putFloor(t, s, "v0.31.0") |
| 125 | if f, _ := syncLocalFloor(ctx, s, dd); f == nil || f.MinRelease != "v0.31.0" { |
| 126 | t.Fatalf("raised = %+v", f) |
| 127 | } |
| 128 | if b, _ := os.ReadFile(filepath.Join(dd, moves.FloorMinReleaseFile)); string(b) != "v0.31.0\n" { |
| 129 | t.Fatalf("floor.min-release after a raise = %q", b) |
| 130 | } |
| 131 | |
| 132 | // A malformed min_release is never armed over a good one. |
| 133 | s.Delete(ctx, moves.FloorKey) |
| 134 | putFloor(t, s, "v0.32.0-rc.1") |
| 135 | if f, err := syncLocalFloor(ctx, s, dd); err == nil || f == nil || f.MinRelease != "v0.31.0" { |
| 136 | t.Fatalf("malformed = %+v, %v", f, err) |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | func TestJudgePrevWritesTheVerdictOnlyAtOrAboveTheFloorWithoutAHold(t *testing.T) { |
| 141 | setup := func(t *testing.T, floor, prev string) (opt, dd string) { |
| 142 | opt, dd = t.TempDir(), t.TempDir() |
| 143 | os.WriteFile(systemPrevPath(opt), []byte("#!/bin/sh\n"), 0o755) |
| 144 | if prev != "" { |
| 145 | if err := writePrevVersion(opt, prev); err != nil { |
| 146 | t.Fatal(err) |
| 147 | } |
| 148 | } |
| 149 | if floor != "" { |
| 150 | armLocalFloor(t, dd, floor) |
| 151 | } |
| 152 | return opt, dd |
| 153 | } |
| 154 | verdict := func(opt string) (string, bool) { |
| 155 | b, err := os.ReadFile(filepath.Join(opt, moves.PrevAllowedFile)) |
| 156 | return string(b), err == nil |
| 157 | } |
| 158 | |
| 159 | t.Run("at the floor", func(t *testing.T) { |
| 160 | opt, dd := setup(t, "v0.30.0", "v0.30.0") |
| 161 | if err := judgePrevWith(opt, dd, noHolds, nil); err != nil { |
| 162 | t.Fatal(err) |
| 163 | } |
| 164 | v, ok := verdict(opt) |
| 165 | line, _ := os.ReadFile(filepath.Join(dd, moves.FloorMinReleaseFile)) |
| 166 | // Byte-identical, because the units compare them with cmp. |
| 167 | if !ok || !bytes.Equal([]byte(v), line) { |
| 168 | t.Fatalf("verdict %q (%v), floor line %q", v, ok, line) |
| 169 | } |
| 170 | }) |
| 171 | for _, tc := range []struct { |
| 172 | name, floor, prev string |
| 173 | holds holdFreezer |
| 174 | }{ |
| 175 | {"below the floor", "v0.30.0", "v0.29.0", noHolds}, |
| 176 | {"a prerelease", "v0.30.0", "v0.30.0-rc.1", noHolds}, |
| 177 | {"no sidecar", "v0.30.0", "", noHolds}, |
| 178 | {"a held row", "v0.30.0", "v0.31.0", oneHold}, |
| 179 | {"the registry unreadable", "v0.30.0", "v0.31.0", unreadableHolds}, |
| 180 | } { |
| 181 | t.Run(tc.name, func(t *testing.T) { |
| 182 | opt, dd := setup(t, tc.floor, tc.prev) |
| 183 | // A verdict from an earlier judgement is removed, not left. |
| 184 | writeFileDurable(filepath.Join(opt, moves.PrevAllowedFile), []byte(tc.floor+"\n"), 0o644) |
| 185 | judgePrevWith(opt, dd, tc.holds, nil) |
| 186 | if v, ok := verdict(opt); ok { |
| 187 | t.Fatalf("verdict %q left for %s", v, tc.name) |
| 188 | } |
| 189 | }) |
| 190 | } |
| 191 | t.Run("no carlos.prev", func(t *testing.T) { |
| 192 | opt, dd := setup(t, "v0.30.0", "v0.31.0") |
| 193 | os.Remove(systemPrevPath(opt)) |
| 194 | writeFileDurable(filepath.Join(opt, moves.PrevAllowedFile), []byte("v0.30.0\n"), 0o644) |
| 195 | if err := judgePrevWith(opt, dd, noHolds, nil); err != nil { |
| 196 | t.Fatal(err) |
| 197 | } |
| 198 | if _, ok := verdict(opt); ok { |
| 199 | t.Fatal("verdict left with no carlos.prev") |
| 200 | } |
| 201 | }) |
| 202 | t.Run("the sidecar is learned when missing", func(t *testing.T) { |
| 203 | opt, dd := setup(t, "v0.30.0", "") |
| 204 | asked := 0 |
| 205 | if err := judgePrevWith(opt, dd, noHolds, func() string { asked++; return "v0.30.1" }); err != nil { |
| 206 | t.Fatal(err) |
| 207 | } |
| 208 | if b, _ := os.ReadFile(filepath.Join(opt, moves.PrevVersionFile)); string(b) != "v0.30.1\n" || asked != 1 { |
| 209 | t.Fatalf("sidecar %q after %d asks", b, asked) |
| 210 | } |
| 211 | if _, ok := verdict(opt); !ok { |
| 212 | t.Fatal("no verdict for a learned version at or above the floor") |
| 213 | } |
| 214 | }) |
| 215 | } |
| 216 | |
| 217 | // No floor armed: judgePrev consults neither the registry nor carlos.prev, |
| 218 | // and leaves no verdict (the units ignore it then anyway). |
| 219 | func TestJudgePrevIsDormantWithoutAFloor(t *testing.T) { |
| 220 | opt, dd := t.TempDir(), t.TempDir() |
| 221 | os.WriteFile(systemPrevPath(opt), []byte("#!/bin/sh\n"), 0o755) |
| 222 | consulted := false |
| 223 | holds := func(fn func([]registry.Route) error) error { consulted = true; return fn(nil) } |
| 224 | ran := false |
| 225 | if err := judgePrevWith(opt, dd, holds, func() string { ran = true; return "v0.29.0" }); err != nil { |
| 226 | t.Fatal(err) |
| 227 | } |
| 228 | if consulted || ran { |
| 229 | t.Fatalf("unarmed judge consulted the registry (%v) or ran carlos.prev (%v)", consulted, ran) |
| 230 | } |
| 231 | for _, name := range []string{moves.PrevAllowedFile, moves.PrevVersionFile} { |
| 232 | if _, err := os.Stat(filepath.Join(opt, name)); err == nil { |
| 233 | t.Errorf("unarmed judge wrote %s", name) |
| 234 | } |
| 235 | } |
| 236 | } |
| 237 | |
| 238 | // The agent's hourly refresh is syncLocalFloor on the agent's own store |
| 239 | // handle; the edges file then publishes what it wrote. |
| 240 | func TestMovesFloorRefreshArmsFromTheStore(t *testing.T) { |
| 241 | dd, s := t.TempDir(), release.NewDirStore(t.TempDir()) |
| 242 | var said []string |
| 243 | logf := func(format string, args ...any) { said = append(said, format) } |
| 244 | movesFloorRefresh(context.Background(), s, dd, logf) |
| 245 | if f, _ := readLocalFloor(dd); f != nil { |
| 246 | t.Fatalf("no object, yet armed %+v", f) |
| 247 | } |
| 248 | putFloor(t, s, "v0.30.0") |
| 249 | movesFloorRefresh(context.Background(), s, dd, logf) |
| 250 | if f, _ := readLocalFloor(dd); f == nil || f.MinRelease != "v0.30.0" { |
| 251 | t.Fatalf("refresh did not arm: %+v", f) |
| 252 | } |
| 253 | movesFloorRefresh(context.Background(), unreadableStore{s}, dd, logf) |
| 254 | if len(said) == 0 { |
| 255 | t.Fatal("a failed refresh said nothing") |
| 256 | } |
| 257 | } |
| 258 | |
| 259 | // The floor never goes down on its own (fix round 1, finding 3). Lowering a |
| 260 | // floor is an operator's manual step on each box: delete the two local files. |
| 261 | func TestSyncLocalFloorRefusesARegression(t *testing.T) { |
| 262 | ctx := context.Background() |
| 263 | dd, s := t.TempDir(), release.NewDirStore(t.TempDir()) |
| 264 | armLocalFloor(t, dd, "v0.31.0") |
| 265 | putFloor(t, s, "v0.30.0") |
| 266 | if f, err := syncLocalFloor(ctx, s, dd); err == nil || f == nil || f.MinRelease != "v0.31.0" { |
| 267 | t.Fatalf("a lower min_release = %+v, %v", f, err) |
| 268 | } |
| 269 | // A lower protocol minimum at the same release is a regression too. |
| 270 | if err := writeLocalFloor(dd, moves.Floor{Min: moves.ProtocolVersion + 1, MinRelease: "v0.31.0"}); err != nil { |
| 271 | t.Fatal(err) |
| 272 | } |
| 273 | s.Delete(ctx, moves.FloorKey) |
| 274 | putFloor(t, s, "v0.31.0") |
| 275 | if f, err := syncLocalFloor(ctx, s, dd); err == nil || f == nil || f.Min != moves.ProtocolVersion+1 { |
| 276 | t.Fatalf("a lower min = %+v, %v", f, err) |
| 277 | } |
| 278 | if b, _ := os.ReadFile(filepath.Join(dd, moves.FloorMinReleaseFile)); string(b) != "v0.31.0\n" { |
| 279 | t.Fatalf("floor.min-release = %q", b) |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | // racingStore answers an old floor, but only after a newer refresh has run |
| 284 | // to completion: an older observation finishing last. |
| 285 | type racingStore struct { |
| 286 | release.Store |
| 287 | newer func() |
| 288 | } |
| 289 | |
| 290 | func (r racingStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 291 | r.newer() |
| 292 | return r.Store.Get(ctx, key) |
| 293 | } |
| 294 | |
| 295 | func TestAnOlderRefreshFinishingLastDoesNotLowerTheFloor(t *testing.T) { |
| 296 | ctx := context.Background() |
| 297 | dd := t.TempDir() |
| 298 | old, newer := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir()) |
| 299 | putFloor(t, old, "v0.30.0") |
| 300 | putFloor(t, newer, "v0.31.0") |
| 301 | race := racingStore{Store: old, newer: func() { |
| 302 | if f, err := syncLocalFloor(ctx, newer, dd); err != nil || f == nil || f.MinRelease != "v0.31.0" { |
| 303 | t.Errorf("newer refresh = %+v, %v", f, err) |
| 304 | } |
| 305 | }} |
| 306 | syncLocalFloor(ctx, race, dd) |
| 307 | if f, _ := readLocalFloor(dd); f == nil || f.MinRelease != "v0.31.0" { |
| 308 | t.Fatalf("the older refresh lowered the floor to %+v", f) |
| 309 | } |
| 310 | if b, _ := os.ReadFile(filepath.Join(dd, moves.FloorMinReleaseFile)); string(b) != "v0.31.0\n" { |
| 311 | t.Fatalf("floor.min-release = %q", b) |
| 312 | } |
| 313 | } |
| 314 | |
| 315 | // Root's apply and the agent's refresh write the floor under one lock file |
| 316 | // in .moves, so neither sees the other's half-written pair. |
| 317 | func TestFloorWritersTakeTheFloorLock(t *testing.T) { |
| 318 | t.Parallel() |
| 319 | ctx := context.Background() |
| 320 | dd, s := t.TempDir(), release.NewDirStore(t.TempDir()) |
| 321 | armLocalFloor(t, dd, "v0.30.0") |
| 322 | putFloor(t, s, "v0.31.0") |
| 323 | held, free := make(chan struct{}), make(chan struct{}) |
| 324 | go withFloorLock(dd, false, func(*movesDir) error { |
| 325 | close(held) |
| 326 | <-free |
| 327 | return nil |
| 328 | }) |
| 329 | <-held |
| 330 | done := make(chan struct{}) |
| 331 | go func() { syncLocalFloor(ctx, s, dd); close(done) }() |
| 332 | select { |
| 333 | case <-done: |
| 334 | t.Fatal("a floor write finished while another writer held the lock") |
| 335 | case <-time.After(300 * time.Millisecond): |
| 336 | } |
| 337 | if f, _ := readLocalFloor(dd); f == nil || f.MinRelease != "v0.30.0" { |
| 338 | t.Fatalf("written under another writer's lock: %+v", f) |
| 339 | } |
| 340 | close(free) |
| 341 | <-done |
| 342 | if f, _ := readLocalFloor(dd); f == nil || f.MinRelease != "v0.31.0" { |
| 343 | t.Fatalf("after the lock = %+v", f) |
| 344 | } |
| 345 | } |
| 346 | |
| 347 | // Readers trust floor.json alone; floor.min-release is written first, so |
| 348 | // after a crash between the two it can only be ahead, and only the units |
| 349 | // read it (where ahead means no verdict matches: the current binary runs). |
| 350 | func TestFloorReadersTrustOnlyTheJSON(t *testing.T) { |
| 351 | dd, opt := t.TempDir(), t.TempDir() |
| 352 | armLocalFloor(t, dd, "v0.30.0") |
| 353 | os.WriteFile(filepath.Join(dd, moves.FloorMinReleaseFile), []byte("v0.31.0\n"), 0o644) |
| 354 | if f, _ := readLocalFloor(dd); f == nil || f.MinRelease != "v0.30.0" { |
| 355 | t.Fatalf("readLocalFloor = %+v", f) |
| 356 | } |
| 357 | if got := withMovesFields(edgesFile{}, dd, opt).MovesFloorMinRelease; got != "v0.30.0" { |
| 358 | t.Fatalf("edges file publishes %q, want the JSON's v0.30.0", got) |
| 359 | } |
| 360 | } |
| 361 | |
amadan