| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | "time" |
| 10 | ) |
| 11 | |
| 12 | // The update gate on a claims-on box (CARLOS_HOST_CLAIM). There the edge's |
| 13 | // listener binds only after the agent's whole claim startup, about 22 s |
| 14 | // after the restart, so a fixed 20 s window rolled every update back and |
| 15 | // the rollback too. With claims on the window is 120 s and the gate fails |
| 16 | // fast on a crash; with claims off it is exactly the 20 s gate it was. |
| 17 | |
| 18 | // slowListener makes the fake edge refuse the probe for listenAfter after |
| 19 | // every restart, as a claims-on edge does while the claim assembly runs. |
| 20 | // It returns the times at which each restart was issued. |
| 21 | func slowListener(f *applyFixture, listenAfter time.Duration) *[]time.Time { |
| 22 | var restarts []time.Time |
| 23 | orig := f.a.ops.systemctl |
| 24 | f.a.ops.systemctl = func(args ...string) (string, error) { |
| 25 | if args[0] == "restart" && len(args) > 1 && args[1] == systemEdgeUnit { |
| 26 | restarts = append(restarts, f.now) |
| 27 | } |
| 28 | return orig(args...) |
| 29 | } |
| 30 | f.a.ops.probeEdge = func() error { |
| 31 | if len(restarts) == 0 || f.now.Sub(restarts[len(restarts)-1]) < listenAfter { |
| 32 | return errors.New("dial tcp 127.0.0.1:80: connect: connection refused") |
| 33 | } |
| 34 | return nil |
| 35 | } |
| 36 | return &restarts |
| 37 | } |
| 38 | |
| 39 | func TestSystemGateWaitIsKeyedOnClaimsMode(t *testing.T) { |
| 40 | if got := systemGateWaitFor(false); got != 20*time.Second { |
| 41 | t.Fatalf("claims-off gate window = %s, want the 20 s it always was", got) |
| 42 | } |
| 43 | if got := systemGateWaitFor(true); got != 120*time.Second { |
| 44 | t.Fatalf("claims-on gate window = %s, want 120 s", got) |
| 45 | } |
| 46 | } |
| 47 | |
| 48 | func TestSystemApplyReadsClaimsModeFromTheUnitEnvironment(t *testing.T) { |
| 49 | for _, tc := range []struct { |
| 50 | value string |
| 51 | claims bool |
| 52 | wait time.Duration |
| 53 | }{{"", false, 20 * time.Second}, {" ", false, 20 * time.Second}, {"1", true, 120 * time.Second}, {"on", true, 120 * time.Second}} { |
| 54 | claims, wait := systemApplyGate(func(k string) string { |
| 55 | if k == "CARLOS_HOST_CLAIM" { |
| 56 | return tc.value |
| 57 | } |
| 58 | return "" |
| 59 | }) |
| 60 | if claims != tc.claims || wait != tc.wait { |
| 61 | t.Fatalf("CARLOS_HOST_CLAIM=%q: claims=%t wait=%s, want %t %s", tc.value, claims, wait, tc.claims, tc.wait) |
| 62 | } |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | func TestClaimsOffGateIsUnchangedByAnEdgeThatListensAt22s(t *testing.T) { |
| 67 | // Ruling 6: a claims-off box sees no change. The same 22 s edge rolls |
| 68 | // back, and its 22 s previous build fails its gate too, as on main. |
| 69 | f, req := applyHappy(t) |
| 70 | slowListener(f, 22*time.Second) |
| 71 | if err := f.a.Run(context.Background(), false); err == nil { |
| 72 | t.Fatal("a claims-off 20 s gate passed an edge that listened at 22 s") |
| 73 | } |
| 74 | r := f.result() |
| 75 | if r == nil || r.Outcome != systemOutcomeRollbackFailed || !r.matches("stable", "v2", req.PromotedAt) { |
| 76 | t.Fatalf("result = %+v, want rollback-failed as on main", r) |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | func TestClaimsOffGateWaitsItsFullWindowThroughACrash(t *testing.T) { |
| 81 | // Claims off never fails fast: a crash-restarting edge still gets the |
| 82 | // full 20 s, and the gate's systemctl calls are only is-active. |
| 83 | f := newApplyFixture(t, "stable", "v1") |
| 84 | f.edgeUp = false |
| 85 | f.nrestarts = "3" |
| 86 | start := f.now |
| 87 | if err := f.a.gate(context.Background(), "v1"); err == nil { |
| 88 | t.Fatal("gate passed a failed edge") |
| 89 | } |
| 90 | if waited := f.now.Sub(start); waited != 20*time.Second { |
| 91 | t.Fatalf("claims-off gate gave up after %s, want exactly 20 s", waited) |
| 92 | } |
| 93 | for _, c := range f.calls { |
| 94 | if strings.HasPrefix(c, "systemctl show") { |
| 95 | t.Fatalf("claims-off gate read %q inside its wait; it must not", c) |
| 96 | } |
| 97 | } |
| 98 | } |
| 99 | |
| 100 | func TestClaimsOnGatePassesAnEdgeThatListensAt22s(t *testing.T) { |
| 101 | f, req := applyHappy(t) |
| 102 | f.a.claims = true |
| 103 | restarts := slowListener(f, 22*time.Second) |
| 104 | if err := f.a.Run(context.Background(), false); err != nil { |
| 105 | t.Fatalf("claims-on gate failed a still-starting edge: %v", err) |
| 106 | } |
| 107 | r := f.result() |
| 108 | if r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) { |
| 109 | t.Fatalf("result = %+v", r) |
| 110 | } |
| 111 | if len(*restarts) != 1 { |
| 112 | t.Fatalf("edge restarted %d times, want 1 (no rollback)", len(*restarts)) |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | func TestClaimsOnGateWaitsUpTo120s(t *testing.T) { |
| 117 | f := newApplyFixture(t, "stable", "v1") |
| 118 | f.a.claims = true |
| 119 | f.probeErr = errors.New("connection refused") |
| 120 | start := f.now |
| 121 | err := f.a.gate(context.Background(), "v1") |
| 122 | if err == nil || !strings.Contains(err.Error(), "connection refused") { |
| 123 | t.Fatalf("gate = %v, want the probe's error at the deadline", err) |
| 124 | } |
| 125 | if waited := f.now.Sub(start); waited != 120*time.Second { |
| 126 | t.Fatalf("claims-on gate gave up after %s, want 120 s", waited) |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | func TestClaimsOnGateFailsFastOnACrashRestart(t *testing.T) { |
| 131 | f := newApplyFixture(t, "stable", "v1") |
| 132 | f.a.claims = true |
| 133 | f.probeErr = errors.New("connection refused") |
| 134 | f.nrestarts = "1" |
| 135 | start := f.now |
| 136 | err := f.a.gate(context.Background(), "v1") |
| 137 | if err == nil || !strings.Contains(err.Error(), "NRestarts=1") { |
| 138 | t.Fatalf("gate = %v, want a fail-fast on NRestarts", err) |
| 139 | } |
| 140 | if waited := f.now.Sub(start); waited != 0 { |
| 141 | t.Fatalf("claims-on gate waited %s on a crash-restarted edge, want no wait", waited) |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | func TestClaimsOnGateFailsFastOnAFailedOrInactiveEdge(t *testing.T) { |
| 146 | for _, state := range []string{"failed", "inactive"} { |
| 147 | t.Run(state, func(t *testing.T) { |
| 148 | f := newApplyFixture(t, "stable", "v1") |
| 149 | f.a.claims = true |
| 150 | orig := f.a.ops.systemctl |
| 151 | f.a.ops.systemctl = func(args ...string) (string, error) { |
| 152 | if args[0] == "is-active" { |
| 153 | return state + "\n", errors.New("exit status 3") |
| 154 | } |
| 155 | return orig(args...) |
| 156 | } |
| 157 | start := f.now |
| 158 | err := f.a.gate(context.Background(), "v1") |
| 159 | if err == nil || !strings.Contains(err.Error(), "failing fast: carlos-edge is "+state) { |
| 160 | t.Fatalf("gate = %v, want a fail-fast on %s", err, state) |
| 161 | } |
| 162 | if waited := f.now.Sub(start); waited != 0 { |
| 163 | t.Fatalf("claims-on gate waited %s on a %s edge", waited, state) |
| 164 | } |
| 165 | }) |
| 166 | } |
| 167 | } |
| 168 | |
| 169 | func TestClaimsOnGateKeepsWaitingWhileActivating(t *testing.T) { |
| 170 | // "activating" with no restart is a process still starting, not a |
| 171 | // crash: no fail-fast, the gate passes once it answers. |
| 172 | f := newApplyFixture(t, "stable", "v1") |
| 173 | f.a.claims = true |
| 174 | start := f.now |
| 175 | orig := f.a.ops.systemctl |
| 176 | f.a.ops.systemctl = func(args ...string) (string, error) { |
| 177 | if args[0] == "is-active" && f.now.Sub(start) < 30*time.Second { |
| 178 | return "activating\n", errors.New("exit status 3") |
| 179 | } |
| 180 | return orig(args...) |
| 181 | } |
| 182 | if err := f.a.gate(context.Background(), "v1"); err != nil { |
| 183 | t.Fatalf("gate = %v, want a pass once the edge is active", err) |
| 184 | } |
| 185 | } |
| 186 | |
| 187 | func TestClaimsOnRollbackGateUsesTheSameRule(t *testing.T) { |
| 188 | // v2 crashes on start; v1, the previous build, needs 22 s to listen. |
| 189 | // The install's gate fails fast, and the rollback's gate gives v1 the |
| 190 | // claims window, so the rollback succeeds instead of rollback-failed. |
| 191 | f, req := applyHappy(t) |
| 192 | f.a.claims = true |
| 193 | restarts := slowListener(f, 22*time.Second) |
| 194 | orig := f.a.ops.systemctl |
| 195 | f.a.ops.systemctl = func(args ...string) (string, error) { |
| 196 | if args[0] == "is-active" { |
| 197 | if b, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); strings.Contains(string(b), "carlos v2") { |
| 198 | return "failed\n", errors.New("exit status 3") |
| 199 | } |
| 200 | } |
| 201 | return orig(args...) |
| 202 | } |
| 203 | err := f.a.Run(context.Background(), false) |
| 204 | if err == nil { |
| 205 | t.Fatal("a failed gate must exit non-zero") |
| 206 | } |
| 207 | r := f.result() |
| 208 | if r == nil || r.Outcome != systemOutcomeRolledBack || !r.matches("stable", "v2", req.PromotedAt) { |
| 209 | t.Fatalf("result = %+v, want rolled-back", r) |
| 210 | } |
| 211 | if !strings.Contains(r.Reason, "failing fast") { |
| 212 | t.Fatalf("reason = %q, want the install gate's fail-fast", r.Reason) |
| 213 | } |
| 214 | if len(*restarts) != 2 { |
| 215 | t.Fatalf("edge restarted %d times, want install + rollback", len(*restarts)) |
| 216 | } |
| 217 | if d := (*restarts)[1].Sub((*restarts)[0]); d > 5*time.Second { |
| 218 | t.Fatalf("install gate took %s to give up on a failed edge, want fail-fast", d) |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | func TestClaimsOnRollbackGateFailsFastToo(t *testing.T) { |
| 223 | // Both builds crash: the rollback gate fails fast as the install's did, |
| 224 | // rather than waiting 120 s on a dead previous build. |
| 225 | f, req := applyHappy(t) |
| 226 | f.a.claims = true |
| 227 | f.nrestarts = "2" |
| 228 | f.probeErr = errors.New("connection refused") |
| 229 | start := f.now |
| 230 | if err := f.a.Run(context.Background(), false); err == nil { |
| 231 | t.Fatal("a failed gate must exit non-zero") |
| 232 | } |
| 233 | r := f.result() |
| 234 | if r == nil || r.Outcome != systemOutcomeRollbackFailed || !r.matches("stable", "v2", req.PromotedAt) { |
| 235 | t.Fatalf("result = %+v, want rollback-failed", r) |
| 236 | } |
| 237 | if waited := f.now.Sub(start); waited >= 20*time.Second { |
| 238 | t.Fatalf("install and rollback gates took %s between them, want fail-fast on both", waited) |
| 239 | } |
| 240 | } |
| 241 | |
amadan