| 1 | package engines |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "os" |
| 6 | "os/exec" |
| 7 | "reflect" |
| 8 | "sort" |
| 9 | "strconv" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | ) |
| 13 | |
| 14 | // seedsFromEnv is the sweep size: ENGINESIM_SEEDS, else def. |
| 15 | func seedsFromEnv(def int) int { |
| 16 | if s := os.Getenv("ENGINESIM_SEEDS"); s != "" { |
| 17 | if n, err := strconv.Atoi(s); err == nil && n > 0 { |
| 18 | return n |
| 19 | } |
| 20 | } |
| 21 | return def |
| 22 | } |
| 23 | |
| 24 | // TestEngines is the sweep. ENGINESIM_SEEDS sets the seed count (the gate's |
| 25 | // default is 200); ENGINESIM_SEED=n runs one seed and prints its trace. |
| 26 | func TestEngines(t *testing.T) { |
| 27 | if s := os.Getenv("ENGINESIM_SEED"); s != "" { |
| 28 | seed, err := strconv.ParseInt(s, 10, 64) |
| 29 | if err != nil { |
| 30 | t.Fatalf("ENGINESIM_SEED=%q: %v", s, err) |
| 31 | } |
| 32 | w := runSeed(seed, true) |
| 33 | for _, line := range w.trace { |
| 34 | t.Log(line) |
| 35 | } |
| 36 | for _, v := range w.violations { |
| 37 | t.Errorf("seed %d: %s", seed, v) |
| 38 | } |
| 39 | return |
| 40 | } |
| 41 | n := seedsFromEnv(200) |
| 42 | failed := 0 |
| 43 | count := map[string]int{} |
| 44 | firstSeed := map[string]int64{} |
| 45 | stats, done, events := map[string]int{}, map[string]int{}, map[string]int{} |
| 46 | residuals := 0 |
| 47 | for seed := int64(1); seed <= int64(n); seed++ { |
| 48 | w := runSeed(seed, false) |
| 49 | for k, v := range w.stats { |
| 50 | stats[k] += v |
| 51 | } |
| 52 | for k, v := range w.done { |
| 53 | done[k] += v |
| 54 | } |
| 55 | for k, v := range w.events { |
| 56 | events[k] += v |
| 57 | } |
| 58 | residuals += len(w.residuals) |
| 59 | if len(w.violations) == 0 { |
| 60 | continue |
| 61 | } |
| 62 | failed++ |
| 63 | for _, v := range w.violations { |
| 64 | id := invariantOf(v) |
| 65 | count[id]++ |
| 66 | if _, ok := firstSeed[id]; !ok { |
| 67 | firstSeed[id] = seed |
| 68 | t.Logf("seed %d: %s", seed, v) |
| 69 | } |
| 70 | } |
| 71 | } |
| 72 | t.Logf("%d seeds, %d failed, %d residuals; mutated=%q", n, failed, residuals, mutated) |
| 73 | t.Logf("actions chosen: %v", stats) |
| 74 | t.Logf("actions completed: %v", done) |
| 75 | t.Logf("events: %v", events) |
| 76 | ids := make([]string, 0, len(count)) |
| 77 | for id := range count { |
| 78 | ids = append(ids, id) |
| 79 | } |
| 80 | sort.Strings(ids) |
| 81 | for _, id := range ids { |
| 82 | t.Logf(" %4d x %s (first seed %d)", count[id], id, firstSeed[id]) |
| 83 | } |
| 84 | if failed > 0 { |
| 85 | t.Fatalf("%d of %d seeds violated an invariant", failed, n) |
| 86 | } |
| 87 | // the default world sets no residual flag, so it never generates one of |
| 88 | // the spec's stated residuals (TestResidualsAreCountedNotFailed) |
| 89 | if residuals > 0 { |
| 90 | t.Fatalf("the default world recorded %d residuals; only a test's flag may generate one", residuals) |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | // TestSeedDeterministic: one seed gives one trace, or no reported seed can |
| 95 | // be replayed. |
| 96 | func TestSeedDeterministic(t *testing.T) { |
| 97 | for _, verbose := range []bool{false, true} { |
| 98 | a, b := runSeed(7, verbose), runSeed(7, verbose) |
| 99 | if len(a.trace) == 0 { |
| 100 | t.Fatalf("verbose=%v: the trace is empty, so equality proves nothing", verbose) |
| 101 | } |
| 102 | if !reflect.DeepEqual(a.trace, b.trace) { |
| 103 | t.Fatalf("verbose=%v: two runs of seed 7 differ", verbose) |
| 104 | } |
| 105 | if !reflect.DeepEqual(a.stats, b.stats) || !reflect.DeepEqual(a.done, b.done) || !reflect.DeepEqual(a.events, b.events) || !reflect.DeepEqual(a.violations, b.violations) { |
| 106 | t.Fatalf("verbose=%v: two runs of seed 7 differ in stats or violations", verbose) |
| 107 | } |
| 108 | } |
| 109 | if reflect.DeepEqual(newWorld(7, false).builds, newWorld(8, false).builds) && |
| 110 | reflect.DeepEqual(newWorld(7, false).builds, newWorld(9, false).builds) && |
| 111 | reflect.DeepEqual(newWorld(7, false).builds, newWorld(10, false).builds) { |
| 112 | t.Fatal("builds do not vary with the seed") |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | // TestMain handles ENGINESIM_LIST=1 before any test runs: it prints the rules |
| 117 | // a run would check, one per line, to stdout and exits 0 without running a |
| 118 | // test (the mutation harness reads this list). |
| 119 | func TestMain(m *testing.M) { |
| 120 | if os.Getenv("ENGINESIM_LIST") == "1" { |
| 121 | for _, r := range allRules { |
| 122 | fmt.Println(r) |
| 123 | } |
| 124 | os.Exit(0) |
| 125 | } |
| 126 | os.Exit(m.Run()) |
| 127 | } |
| 128 | |
| 129 | // TestListPrintsRulesAndRunsNothing runs this test binary as a subprocess |
| 130 | // with ENGINESIM_LIST=1 and requires that its output is exactly the rule |
| 131 | // list and that no test ran. A control run without the variable shows the |
| 132 | // "=== RUN" marker is what a test run prints. |
| 133 | func TestListPrintsRulesAndRunsNothing(t *testing.T) { |
| 134 | run := func(list bool) string { |
| 135 | cmd := exec.Command(os.Args[0], "-test.v", "-test.run=TestOnMutation") |
| 136 | cmd.Env = os.Environ() |
| 137 | for i := 0; i < len(cmd.Env); i++ { |
| 138 | if strings.HasPrefix(cmd.Env[i], "ENGINESIM_LIST=") { |
| 139 | cmd.Env = append(cmd.Env[:i], cmd.Env[i+1:]...) |
| 140 | i-- |
| 141 | } |
| 142 | } |
| 143 | if list { |
| 144 | cmd.Env = append(cmd.Env, "ENGINESIM_LIST=1") |
| 145 | } |
| 146 | out, err := cmd.Output() |
| 147 | if err != nil { |
| 148 | t.Fatalf("list=%v: %v", list, err) |
| 149 | } |
| 150 | return string(out) |
| 151 | } |
| 152 | if control := run(false); !strings.Contains(control, "=== RUN") { |
| 153 | t.Fatalf("control run printed no test markers: %q", control) |
| 154 | } |
| 155 | want := "" |
| 156 | for _, r := range allRules { |
| 157 | want += r + "\n" |
| 158 | } |
| 159 | if got := run(true); got != want { |
| 160 | t.Fatalf("ENGINESIM_LIST=1 output = %q, want exactly the rule list %q", got, want) |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | // TestNewWorldShape pins the starting world the brief describes. |
| 165 | func TestNewWorldShape(t *testing.T) { |
| 166 | w := newWorld(1, false) |
| 167 | if len(w.boxes) != 2 || w.boxes[0].ID != 0 || w.boxes[1].ID != 1 { |
| 168 | t.Fatalf("want boxes 0 and 1, got %d", len(w.boxes)) |
| 169 | } |
| 170 | const eng = "eng.acct.apps.example" |
| 171 | r := w.boxes[0].Rows[eng] |
| 172 | if r == nil || !r.Platform || r.Kind != "instance" || r.Owner != "eng" || r.Backend == "" { |
| 173 | t.Fatalf("box 0 lacks the minted platform row %s: %+v", eng, r) |
| 174 | } |
| 175 | if got := w.group(w.boxes[0], eng); got != "eng" { |
| 176 | t.Fatalf("group of the minted row = %q, want eng", got) |
| 177 | } |
| 178 | if m := w.boxes[0].Mark[eng]; m != markInternal { |
| 179 | t.Fatalf("the engine row's mark = %v, want markInternal", m) |
| 180 | } |
| 181 | s := w.boxes[1].Rows["shop"] |
| 182 | if s == nil || s.Platform || s.Owner != "" || s.Backend == "" { |
| 183 | t.Fatalf("box 1 lacks the public app row shop: %+v", s) |
| 184 | } |
| 185 | if got := w.group(w.boxes[1], "shop"); got != "shop" { |
| 186 | t.Fatalf("group of an unminted row = %q, want its host", got) |
| 187 | } |
| 188 | if m := w.boxes[1].Mark["shop"]; m != markPublic { |
| 189 | t.Fatalf("shop's mark = %v, want markPublic", m) |
| 190 | } |
| 191 | for v := 1; v <= 4; v++ { |
| 192 | if b, ok := w.builds[v]; !ok || b.Ver != v { |
| 193 | t.Fatalf("build %d missing or mislabelled: %+v", v, b) |
| 194 | } |
| 195 | } |
| 196 | if len(w.builds) != 4 { |
| 197 | t.Fatalf("want builds 1 to 4, got %d", len(w.builds)) |
| 198 | } |
| 199 | for _, b := range w.boxes { |
| 200 | if b.Rows == nil || b.Mark == nil || b.Live == nil || b.Good == nil || b.Staged == nil { |
| 201 | t.Fatalf("box %d has a nil map", b.ID) |
| 202 | } |
| 203 | } |
| 204 | // a service row and a static row group by their own host |
| 205 | b := w.boxes[1] |
| 206 | b.Rows["svc"] = &row{Host: "svc", Kind: "service", Owner: "x"} |
| 207 | if got := w.group(b, "svc"); got != "svc" { |
| 208 | t.Fatalf("group of a service row = %q, want svc", got) |
| 209 | } |
| 210 | } |
| 211 | |
| 212 | // TestStepLoop: with no actions a step does nothing but advance w.step; with |
| 213 | // actions registered each run is counted under the action's name (in done |
| 214 | // only when it reports a completed transition), the walk is weighted, and |
| 215 | // 400 chaos plus 200 calm steps run. |
| 216 | func TestStepLoop(t *testing.T) { |
| 217 | saved := actions |
| 218 | defer func() { actions = saved }() |
| 219 | |
| 220 | actions = nil |
| 221 | w := runSeed(3, false) |
| 222 | if w.step != chaosSteps+calmSteps { |
| 223 | t.Fatalf("w.step = %d, want %d", w.step, chaosSteps+calmSteps) |
| 224 | } |
| 225 | if len(w.stats) != 0 || len(w.violations) != 0 { |
| 226 | t.Fatalf("no actions: stats %v violations %v", w.stats, w.violations) |
| 227 | } |
| 228 | |
| 229 | calls := map[string]int{} |
| 230 | var chaosCalls, calmCalls int |
| 231 | phase := func(w *world) { |
| 232 | if w.calm() { |
| 233 | calmCalls++ |
| 234 | } else { |
| 235 | chaosCalls++ |
| 236 | } |
| 237 | } |
| 238 | actions = []action{ |
| 239 | {name: "alpha", weight: 3, run: func(w *world) bool { |
| 240 | calls["alpha"]++ |
| 241 | phase(w) |
| 242 | return true |
| 243 | }}, |
| 244 | {name: "beta", weight: 1, run: func(w *world) bool { |
| 245 | calls["beta"]++ |
| 246 | phase(w) |
| 247 | return false // chosen, but found nothing to do |
| 248 | }}, |
| 249 | {name: "never", weight: 0, run: func(w *world) bool { t.Error("a zero-weight action ran"); return true }}, |
| 250 | } |
| 251 | w = runSeed(3, false) |
| 252 | if w.stats["alpha"] != calls["alpha"] || w.stats["beta"] != calls["beta"] { |
| 253 | t.Fatalf("stats %v do not match calls %v", w.stats, calls) |
| 254 | } |
| 255 | if w.done["alpha"] != calls["alpha"] || w.done["beta"] != 0 { |
| 256 | t.Fatalf("done %v: want alpha's every run and none of beta's (calls %v)", w.done, calls) |
| 257 | } |
| 258 | if calls["alpha"]+calls["beta"] != chaosSteps+calmSteps { |
| 259 | t.Fatalf("each step should run one action: %v", calls) |
| 260 | } |
| 261 | if calls["alpha"] < 2*calls["beta"] { |
| 262 | t.Fatalf("weights ignored: %v", calls) |
| 263 | } |
| 264 | if chaosCalls != chaosSteps || calmCalls != calmSteps || chaosSteps != 400 || calmSteps != 200 { |
| 265 | t.Fatalf("phase split: chaos %d calm %d, want 400 and 200", chaosCalls, calmCalls) |
| 266 | } |
| 267 | if _, ok := w.stats["never"]; ok { |
| 268 | t.Fatal("a zero-weight action was counted") |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | // TestFailOncePerInvariant: fail records the first violation of each |
| 273 | // invariant id and ignores the repeats. |
| 274 | func TestFailOncePerInvariant(t *testing.T) { |
| 275 | w := newWorld(1, false) |
| 276 | w.fail("INV-X", "first %d", 1) |
| 277 | w.fail("INV-X", "second %d", 2) |
| 278 | w.fail("INV-Y", "other") |
| 279 | if len(w.violations) != 2 { |
| 280 | t.Fatalf("violations = %v, want two", w.violations) |
| 281 | } |
| 282 | if !strings.HasPrefix(w.violations[0], "INV-X: first 1") || !strings.HasPrefix(w.violations[1], "INV-Y: other") { |
| 283 | t.Fatalf("violations = %v", w.violations) |
| 284 | } |
| 285 | if invariantOf(w.violations[0]) != "INV-X" { |
| 286 | t.Fatalf("invariantOf = %q", invariantOf(w.violations[0])) |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | // TestOnMutation: on is false only for a rule named in the mutation set. |
| 291 | func TestOnMutation(t *testing.T) { |
| 292 | saved := mutated |
| 293 | defer func() { mutated = saved }() |
| 294 | mutated = "" |
| 295 | if !on("any-rule") { |
| 296 | t.Fatal("no mutation: every rule is on") |
| 297 | } |
| 298 | mutated = "a-rule,b-rule" |
| 299 | if on("a-rule") || on("b-rule") || !on("c-rule") { |
| 300 | t.Fatal("only the named rules are off") |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | // TestEveryActionFires: over 200 seeds of the unmutated walk every registered |
| 305 | // action completes its transition at least once (chosen is not enough: an |
| 306 | // action that always finds nothing to do exercises nothing), and public |
| 307 | // requests and engine calls really reach processes. A rule that guards an |
| 308 | // action no seed fires is untested however green the sweep looks (Review |
| 309 | // Focus 2; final review, minor 3). |
| 310 | func TestEveryActionFires(t *testing.T) { |
| 311 | saved := mutated |
| 312 | mutated = "" |
| 313 | defer func() { mutated = saved }() |
| 314 | if len(actions) == 0 { |
| 315 | t.Fatal("no actions registered") |
| 316 | } |
| 317 | names := map[string]bool{} |
| 318 | for _, a := range actions { |
| 319 | if names[a.name] { |
| 320 | t.Errorf("action %q registered twice", a.name) |
| 321 | } |
| 322 | names[a.name] = true |
| 323 | if a.weight <= 0 { |
| 324 | t.Errorf("action %q has weight %d: the walk never picks it", a.name, a.weight) |
| 325 | } |
| 326 | } |
| 327 | done, events := map[string]int{}, map[string]int{} |
| 328 | for seed := int64(1); seed <= 200; seed++ { |
| 329 | w := runSeed(seed, false) |
| 330 | for k, v := range w.done { |
| 331 | done[k] += v |
| 332 | } |
| 333 | for k, v := range w.events { |
| 334 | events[k] += v |
| 335 | } |
| 336 | } |
| 337 | for _, a := range actions { |
| 338 | if done[a.name] == 0 { |
| 339 | t.Errorf("action %q never completed a transition over 200 seeds", a.name) |
| 340 | } |
| 341 | } |
| 342 | for _, e := range []string{"delivered", "engine-call-reached"} { |
| 343 | if events[e] == 0 { |
| 344 | t.Errorf("no %s over 200 seeds", e) |
| 345 | } |
| 346 | } |
| 347 | t.Logf("completed: %v; events: %v", done, events) |
| 348 | } |
| 349 | |
amadan