| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/hostclaim" |
| 12 | "github.com/carlosframework/platform/internal/registry" |
| 13 | "github.com/carlosframework/platform/internal/release" |
| 14 | ) |
| 15 | |
| 16 | // seedInstance upserts a ready, provisioned exec instance: a relocatable |
| 17 | // host (spec 1 §2). |
| 18 | func seedInstance(t *testing.T, reg *registry.Registry, host string) { |
| 19 | t.Helper() |
| 20 | if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Socket: "/run/carlos/" + host + ".sock", |
| 21 | Provisioned: true, App: "jam", Account: "bab", Status: "ready"}); err != nil { |
| 22 | t.Fatal(err) |
| 23 | } |
| 24 | } |
| 25 | |
| 26 | // claimsBook answers claim reads from a map; a missing host is an absent |
| 27 | // claim, a host in errs fails. |
| 28 | type claimsBook struct { |
| 29 | claims map[string]*hostclaim.Claim |
| 30 | errs map[string]bool |
| 31 | reads []string |
| 32 | } |
| 33 | |
| 34 | func (b *claimsBook) read(_ context.Context, host string) (hostclaim.Observed, error) { |
| 35 | b.reads = append(b.reads, host) |
| 36 | if b.errs[host] { |
| 37 | return hostclaim.Observed{}, errors.New("SlowDown") |
| 38 | } |
| 39 | c, ok := b.claims[host] |
| 40 | if !ok { |
| 41 | return hostclaim.Observed{Host: host, Absent: true}, nil |
| 42 | } |
| 43 | return hostclaim.Observed{Host: host, Claim: c, ETag: "e"}, nil |
| 44 | } |
| 45 | |
| 46 | func serving(exec string, stamp uint64) *hostclaim.Claim { |
| 47 | return &hostclaim.Claim{V: 2, Gen: 7, Holder: &hostclaim.Holder{Stamp: stamp}, |
| 48 | Op: hostclaim.Op{Kind: hostclaim.KindServing, ID: "op-1", Exec: exec}} |
| 49 | } |
| 50 | |
| 51 | func publishWithClaims(t *testing.T, reg *registry.Registry, claims *edgesClaims) map[string]edgeHost { |
| 52 | t.Helper() |
| 53 | // These tests are about the hosts; the box's addresses come from a |
| 54 | // fixed source, never from IMDS. nil claims is region HA off. |
| 55 | if claims != nil { |
| 56 | claims.addrs = staticAddresses{Public: "1.2.3.4"} |
| 57 | } |
| 58 | st := release.NewDirStore(t.TempDir()) |
| 59 | var last []byte |
| 60 | if err := edgesFileWriteClaims(context.Background(), st, reg, edgesFile{Box: "annie", BoxID: "annie"}, "1.2.3.4", &last, false, claims); err != nil { |
| 61 | t.Fatal(err) |
| 62 | } |
| 63 | body, _, err := st.Get(context.Background(), edgesFileKey("annie")) |
| 64 | if err != nil { |
| 65 | t.Fatal(err) |
| 66 | } |
| 67 | var f edgesFile |
| 68 | if err := json.Unmarshal(body, &f); err != nil { |
| 69 | t.Fatal(err) |
| 70 | } |
| 71 | return f.Hosts |
| 72 | } |
| 73 | |
| 74 | func TestEdgesFileFollowsTheClaimsExecutor(t *testing.T) { |
| 75 | reg := edgesTestRegistry(t) |
| 76 | for _, h := range []string{"mine.bab.test", "theirs.bab.test", "between.bab.test", "unclaimed.bab.test"} { |
| 77 | seedInstance(t, reg, h) |
| 78 | } |
| 79 | seedRoute(t, reg, "static.example.com") // not relocatable: never read, always home |
| 80 | book := &claimsBook{claims: map[string]*hostclaim.Claim{ |
| 81 | "mine.bab.test": serving("annie", 41), |
| 82 | "theirs.bab.test": serving("bert", 42), |
| 83 | "between.bab.test": serving("", 43), // op.exec none: nobody serves it |
| 84 | }} |
| 85 | hosts := publishWithClaims(t, reg, newEdgesClaims("annie", book.read)) |
| 86 | if h, ok := hosts["mine.bab.test"]; !ok || h.Stamp != 41 || h.Role != "home" { |
| 87 | t.Fatalf("executor's host = %+v, %v; want home stamped 41", h, ok) |
| 88 | } |
| 89 | for _, h := range []string{"theirs.bab.test", "between.bab.test"} { |
| 90 | if _, ok := hosts[h]; ok { |
| 91 | t.Errorf("%s published by a box its claim does not name", h) |
| 92 | } |
| 93 | } |
| 94 | if h, ok := hosts["unclaimed.bab.test"]; !ok || h.Stamp != 0 { |
| 95 | t.Errorf("a host with no claim = %+v, %v; want home, unstamped, as before", h, ok) |
| 96 | } |
| 97 | if h, ok := hosts["static.example.com"]; !ok || h.Stamp != 0 { |
| 98 | t.Errorf("a service route = %+v, %v; want home, unstamped", h, ok) |
| 99 | } |
| 100 | for _, r := range book.reads { |
| 101 | if r == "static.example.com" { |
| 102 | t.Error("read a claim for a route that is not relocatable") |
| 103 | } |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | func TestEdgesFileWithholdsTheAliasOfAWithheldHost(t *testing.T) { |
| 108 | reg := edgesTestRegistry(t) |
| 109 | seedInstance(t, reg, "theirs.bab.test") |
| 110 | if err := reg.UpsertAlias(registry.RouteAlias{Alias: "www.customer.com", Host: "theirs.bab.test", Account: "bab", App: "jam", DNSStatus: "pointing"}); err != nil { |
| 111 | t.Fatal(err) |
| 112 | } |
| 113 | book := &claimsBook{claims: map[string]*hostclaim.Claim{"theirs.bab.test": serving("bert", 42)}} |
| 114 | hosts := publishWithClaims(t, reg, newEdgesClaims("annie", book.read)) |
| 115 | if len(hosts) != 0 { |
| 116 | t.Fatalf("hosts = %v, want none", hosts) |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | // Review Focus 2: a claim that cannot be read never moves a host. It keeps |
| 121 | // the last decision, or, never read, publishes as it did before claims. |
| 122 | func TestEdgesFileKeepsTheLastDecisionWhenAClaimCannotBeRead(t *testing.T) { |
| 123 | reg := edgesTestRegistry(t) |
| 124 | seedInstance(t, reg, "mine.bab.test") |
| 125 | seedInstance(t, reg, "theirs.bab.test") |
| 126 | book := &claimsBook{claims: map[string]*hostclaim.Claim{"mine.bab.test": serving("annie", 41), "theirs.bab.test": serving("bert", 42)}} |
| 127 | clock := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 128 | claims := newEdgesClaims("annie", book.read) |
| 129 | claims.now = func() time.Time { return clock } |
| 130 | publishWithClaims(t, reg, claims) |
| 131 | clock = clock.Add(edgesClaimTTL) |
| 132 | book.errs = map[string]bool{"mine.bab.test": true, "theirs.bab.test": true} |
| 133 | hosts := publishWithClaims(t, reg, claims) |
| 134 | if h, ok := hosts["mine.bab.test"]; !ok || h.Stamp != 41 { |
| 135 | t.Fatalf("mine = %+v, %v: an unreadable claim dropped or unstamped a served host", h, ok) |
| 136 | } |
| 137 | if _, ok := hosts["theirs.bab.test"]; ok { |
| 138 | t.Fatal("an unreadable claim published a host this box was told not to") |
| 139 | } |
| 140 | |
| 141 | fresh := newEdgesClaims("annie", book.read) |
| 142 | if h, ok := publishWithClaims(t, reg, fresh)["mine.bab.test"]; !ok || h.Stamp != 0 { |
| 143 | t.Fatalf("never read = %+v, %v; want home, unstamped", h, ok) |
| 144 | } |
| 145 | } |
| 146 | |
| 147 | func TestEdgesClaimsRereadOnlyAfterTheTTL(t *testing.T) { |
| 148 | book := &claimsBook{claims: map[string]*hostclaim.Claim{"mine.bab.test": serving("annie", 41)}} |
| 149 | clock := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 150 | c := newEdgesClaims("annie", book.read) |
| 151 | c.now = func() time.Time { return clock } |
| 152 | ctx := context.Background() |
| 153 | c.decide(ctx, []string{"mine.bab.test"}) |
| 154 | clock = clock.Add(edgesClaimTTL - time.Second) |
| 155 | c.decide(ctx, []string{"mine.bab.test"}) |
| 156 | if len(book.reads) != 1 { |
| 157 | t.Fatalf("%d reads inside the TTL, want 1", len(book.reads)) |
| 158 | } |
| 159 | book.claims["mine.bab.test"] = serving("bert", 50) |
| 160 | clock = clock.Add(time.Second) |
| 161 | if d := c.decide(ctx, []string{"mine.bab.test"})["mine.bab.test"]; !d.withhold { |
| 162 | t.Fatalf("decision after the TTL = %+v; the move to another box was not seen", d) |
| 163 | } |
| 164 | c.decide(ctx, nil) |
| 165 | if len(c.entries) != 0 { |
| 166 | t.Fatalf("a host no longer served kept its entry: %v", c.entries) |
| 167 | } |
| 168 | } |
| 169 | |
| 170 | func TestEdgesClaimsBoundReadsPerPassNeverReadFirst(t *testing.T) { |
| 171 | book := &claimsBook{claims: map[string]*hostclaim.Claim{}} |
| 172 | clock := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 173 | c := newEdgesClaims("annie", book.read) |
| 174 | c.now = func() time.Time { return clock } |
| 175 | var hosts []string |
| 176 | for i := 0; i < edgesClaimReadsPerPass+10; i++ { |
| 177 | hosts = append(hosts, fmt.Sprintf("h%04d.bab.test", i)) |
| 178 | } |
| 179 | ctx := context.Background() |
| 180 | c.decide(ctx, hosts) |
| 181 | if len(book.reads) != edgesClaimReadsPerPass { |
| 182 | t.Fatalf("first pass read %d claims, want %d", len(book.reads), edgesClaimReadsPerPass) |
| 183 | } |
| 184 | book.reads = nil |
| 185 | c.decide(ctx, hosts) |
| 186 | if len(book.reads) != 10 { |
| 187 | t.Fatalf("second pass read %d claims, want the 10 never read", len(book.reads)) |
| 188 | } |
| 189 | } |
| 190 | |
| 191 | // Astra plan R1-2: an alias and a served catch-all are dialled through |
| 192 | // their primary's installation, so they carry its stamp. Unstamped, a dead |
| 193 | // predecessor's alias would tie with the successor's and the edge would |
| 194 | // keep the first box by name. |
| 195 | func TestEdgesFileStampsAliasesAndCatchAllsWithTheirPrimary(t *testing.T) { |
| 196 | reg := edgesTestRegistry(t) |
| 197 | seedInstance(t, reg, "mine.bab.test") |
| 198 | for _, a := range []registry.RouteAlias{ |
| 199 | {Alias: "www.customer.com", Host: "mine.bab.test", Account: "bab", App: "jam", DNSStatus: "pointing"}, |
| 200 | {Alias: "*.customer.com", Host: "mine.bab.test", Account: "bab", App: "jam", DNSStatus: "pointing"}, |
| 201 | } { |
| 202 | if err := reg.UpsertAlias(a); err != nil { |
| 203 | t.Fatal(err) |
| 204 | } |
| 205 | } |
| 206 | if err := reg.UpsertWildcardDomain(registry.WildcardDomain{Host: "customer.com", Account: "bab", App: "jam"}); err != nil { |
| 207 | t.Fatal(err) |
| 208 | } |
| 209 | book := &claimsBook{claims: map[string]*hostclaim.Claim{"mine.bab.test": serving("annie", 57)}} |
| 210 | hosts := publishWithClaims(t, reg, newEdgesClaims("annie", book.read)) |
| 211 | for _, key := range []string{"mine.bab.test", "www.customer.com", "*.customer.com"} { |
| 212 | if h, ok := hosts[key]; !ok || h.Stamp != 57 || h.Role != "home" { |
| 213 | t.Errorf("%s = %+v, %v; want home at stamp 57", key, h, ok) |
| 214 | } |
| 215 | } |
| 216 | // With no claim reader the file is what it always was: no stamps. |
| 217 | for key, h := range publishWithClaims(t, reg, nil) { |
| 218 | if h.Stamp != 0 { |
| 219 | t.Errorf("%s stamped %d with region HA off", key, h.Stamp) |
| 220 | } |
| 221 | } |
| 222 | } |
| 223 | |
| 224 | // Astra plan R1-4: claims that keep failing go to the back of the queue, |
| 225 | // so they cannot hold every read of every pass. |
| 226 | func TestEdgesClaimsAFailingClaimDoesNotStarveTheRest(t *testing.T) { |
| 227 | book := &claimsBook{claims: map[string]*hostclaim.Claim{"zz-healthy.bab.test": serving("annie", 7)}, errs: map[string]bool{}} |
| 228 | var hosts []string |
| 229 | for i := 0; i < edgesClaimReadsPerPass; i++ { |
| 230 | h := fmt.Sprintf("a%04d.bab.test", i) |
| 231 | book.errs[h] = true |
| 232 | hosts = append(hosts, h) |
| 233 | } |
| 234 | hosts = append(hosts, "zz-healthy.bab.test") |
| 235 | clock := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 236 | c := newEdgesClaims("annie", book.read) |
| 237 | c.now = func() time.Time { return clock } |
| 238 | ctx := context.Background() |
| 239 | c.decide(ctx, hosts) // the 256 failing claims sort first and all fail |
| 240 | clock = clock.Add(edgesClaimPassEvery) |
| 241 | if d, ok := c.decide(ctx, hosts)["zz-healthy.bab.test"]; !ok || d.stamp != 7 { |
| 242 | t.Fatalf("healthy claim = %+v, %v: starved by claims that cannot be read", d, ok) |
| 243 | } |
| 244 | } |
| 245 | |
| 246 | // Astra plan R1-5: the loop's own schedule, with a stable registry, keeps |
| 247 | // every claim of a 2,000-host box fresh. Timer passes alone come every |
| 248 | // five to six minutes and read 256 claims each. |
| 249 | func TestEdgesLoopRefreshesEveryClaimWithinTheTTL(t *testing.T) { |
| 250 | t.Parallel() |
| 251 | const n = 2000 |
| 252 | book := &claimsBook{claims: map[string]*hostclaim.Claim{}} |
| 253 | var hosts []string |
| 254 | for i := 0; i < n; i++ { |
| 255 | h := fmt.Sprintf("h%04d.bab.test", i) |
| 256 | book.claims[h] = serving("annie", uint64(i+1)) |
| 257 | hosts = append(hosts, h) |
| 258 | } |
| 259 | start := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 260 | clock := start |
| 261 | claims := newEdgesClaims("annie", book.read) |
| 262 | claims.now = func() time.Time { return clock } |
| 263 | state := &edgesLoopState{sched: newEdgesFileSchedule(start, func() time.Duration { return 5*time.Minute + 30*time.Second }), claims: claims} |
| 264 | lastRead := map[string]time.Time{} |
| 265 | worst := time.Duration(0) |
| 266 | for clock = start; clock.Before(start.Add(3 * time.Hour)); clock = clock.Add(edgesFileWatchInterval) { |
| 267 | if !state.due(clock, "fp", true) { |
| 268 | continue |
| 269 | } |
| 270 | book.reads = nil |
| 271 | claims.decide(context.Background(), hosts) |
| 272 | for _, h := range book.reads { |
| 273 | lastRead[h] = clock |
| 274 | } |
| 275 | state.done(clock, true) |
| 276 | if clock.Sub(start) < time.Hour { |
| 277 | continue // warm-up: the first reads of every host |
| 278 | } |
| 279 | for _, h := range hosts { |
| 280 | if age := clock.Sub(lastRead[h]); age > worst { |
| 281 | worst = age |
| 282 | } |
| 283 | } |
| 284 | } |
| 285 | if limit := edgesClaimTTL + 5*time.Minute; worst > limit { |
| 286 | t.Fatalf("a claim went %s without a read, want at most %s", worst, limit) |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | // Astra plan R2-1: a claim pass is a publication. While the bucket refuses |
| 291 | // every write, claim passes wait out the schedule's backoff and their own |
| 292 | // failures move it; once writes succeed, the 30-second claim passes resume. |
| 293 | func TestEdgesLoopClaimPassesObeyTheRetryBackoff(t *testing.T) { |
| 294 | t.Parallel() |
| 295 | book := &claimsBook{claims: map[string]*hostclaim.Claim{}} |
| 296 | var hosts []string |
| 297 | for i := 0; i < 4000; i++ { // enough that reads stay due throughout |
| 298 | h := fmt.Sprintf("h%04d.bab.test", i) |
| 299 | book.claims[h] = serving("annie", 1) |
| 300 | hosts = append(hosts, h) |
| 301 | } |
| 302 | start := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 303 | clock := start |
| 304 | claims := newEdgesClaims("annie", book.read) |
| 305 | claims.now = func() time.Time { return clock } |
| 306 | state := &edgesLoopState{sched: newEdgesFileSchedule(start, func() time.Duration { return 5*time.Minute + 30*time.Second }), claims: claims} |
| 307 | failUntil := start.Add(20 * time.Minute) |
| 308 | var during, after []time.Time |
| 309 | for clock = start; clock.Before(failUntil.Add(8 * time.Minute)); clock = clock.Add(edgesFileWatchInterval) { |
| 310 | if !state.due(clock, "fp", true) { |
| 311 | continue |
| 312 | } |
| 313 | claims.decide(context.Background(), hosts) |
| 314 | ok := !clock.Before(failUntil) |
| 315 | if ok { |
| 316 | after = append(after, clock) |
| 317 | } else { |
| 318 | during = append(during, clock) |
| 319 | } |
| 320 | state.done(clock, ok) |
| 321 | } |
| 322 | // Unthrottled, a claim pass every 30 seconds would be 40 attempts in |
| 323 | // 20 minutes. The backoff (2 s doubling to 5 min) plus the timer allows |
| 324 | // about a dozen. |
| 325 | t.Logf("%d attempts while failing, %d after", len(during), len(after)) |
| 326 | if len(during) > 16 { |
| 327 | t.Fatalf("%d publication attempts while every write failed, want at most 16: %v", len(during), during) |
| 328 | } |
| 329 | // Recovery waits out the backoff (at most five minutes), then the |
| 330 | // 30-second claim passes resume. |
| 331 | if len(after) < 3 || after[len(after)-1].Sub(after[len(after)-2]) != edgesClaimPassEvery { |
| 332 | t.Fatalf("passes after writes recovered = %v, want the 30-second claim passes back within eight minutes", after) |
| 333 | } |
| 334 | } |
| 335 | |
| 336 | // Final review 4: a store that never answers cannot stall the edges file. |
| 337 | // The pass has a deadline; a read it cuts short keeps the last decision as |
| 338 | // any failed read does, and the reads not yet made wait for a later pass. |
| 339 | func TestEdgesClaimsPassHasADeadline(t *testing.T) { |
| 340 | book := &claimsBook{claims: map[string]*hostclaim.Claim{"mine.bab.test": serving("annie", 41), "theirs.bab.test": serving("bert", 42)}} |
| 341 | hang := false |
| 342 | read := func(ctx context.Context, host string) (hostclaim.Observed, error) { |
| 343 | if hang { |
| 344 | select { // a Get that blocks until the context ends |
| 345 | case <-ctx.Done(): |
| 346 | return hostclaim.Observed{}, ctx.Err() |
| 347 | case <-time.After(2 * time.Second): // so a missing deadline fails rather than hangs |
| 348 | return hostclaim.Observed{}, errors.New("no deadline") |
| 349 | } |
| 350 | } |
| 351 | return book.read(ctx, host) |
| 352 | } |
| 353 | clock := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 354 | c := newEdgesClaims("annie", read) |
| 355 | c.now = func() time.Time { return clock } |
| 356 | c.passDeadline = 100 * time.Millisecond |
| 357 | c.logf = func(string, ...any) {} |
| 358 | routes := []registry.Route{ |
| 359 | {Host: "mine.bab.test", Kind: "instance", Socket: "/run/carlos/mine.sock", Provisioned: true}, |
| 360 | {Host: "theirs.bab.test", Kind: "instance", Socket: "/run/carlos/theirs.sock", Provisioned: true}, |
| 361 | } |
| 362 | served := func() map[string]edgeHost { |
| 363 | return map[string]edgeHost{"mine.bab.test": {Role: "home"}, "theirs.bab.test": {Role: "home"}} |
| 364 | } |
| 365 | applyEdgesClaims(context.Background(), c, routes, served()) |
| 366 | clock = clock.Add(edgesClaimTTL) |
| 367 | hang = true |
| 368 | hosts := served() |
| 369 | start := time.Now() |
| 370 | applyEdgesClaims(context.Background(), c, routes, hosts) |
| 371 | if took := time.Since(start); took > time.Second { |
| 372 | t.Fatalf("the pass took %s against a store that never answers, want about its 100ms deadline", took) |
| 373 | } |
| 374 | if h, ok := hosts["mine.bab.test"]; !ok || h.Stamp != 41 { |
| 375 | t.Fatalf("mine = %+v, %v: a read cut short by the deadline moved a served host", h, ok) |
| 376 | } |
| 377 | if _, ok := hosts["theirs.bab.test"]; ok { |
| 378 | t.Fatal("a read cut short by the deadline published a host this box was told not to") |
| 379 | } |
| 380 | if d := newEdgesClaims("annie", read).passDeadline; d != 20*time.Second { |
| 381 | t.Fatalf("the default pass deadline is %s, want 20s", d) |
| 382 | } |
| 383 | } |
| 384 | |
| 385 | // Final review 4: a host hostclaim.Key rejects is never read; it publishes |
| 386 | // as it did before claims. |
| 387 | func TestEdgesClaimsNeverReadAHostKeyRejects(t *testing.T) { |
| 388 | const bad = "under_score.bab.test" |
| 389 | if _, err := hostclaim.Key(bad); err == nil { |
| 390 | t.Fatalf("hostclaim.Key accepts %q; the test needs a host it rejects", bad) |
| 391 | } |
| 392 | book := &claimsBook{claims: map[string]*hostclaim.Claim{bad: serving("bert", 42)}} |
| 393 | c := newEdgesClaims("annie", book.read) |
| 394 | routes := []registry.Route{{Host: bad, Kind: "instance", Socket: "/run/carlos/x.sock", Provisioned: true}} |
| 395 | hosts := map[string]edgeHost{bad: {Role: "home"}} |
| 396 | applyEdgesClaims(context.Background(), c, routes, hosts) |
| 397 | if len(book.reads) != 0 { |
| 398 | t.Fatalf("read claims for %v", book.reads) |
| 399 | } |
| 400 | if h, ok := hosts[bad]; !ok || h.Stamp != 0 { |
| 401 | t.Fatalf("%s = %+v, %v; want home, unstamped, as before", bad, h, ok) |
| 402 | } |
| 403 | } |
| 404 | |
| 405 | // Final review 4: claim read failures are logged on the first failing |
| 406 | // pass and every 60th after it, and once when reads recover. |
| 407 | func TestEdgesClaimsLogReadFailuresThrottled(t *testing.T) { |
| 408 | book := &claimsBook{claims: map[string]*hostclaim.Claim{}, errs: map[string]bool{"a.bab.test": true}} |
| 409 | clock := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 410 | c := newEdgesClaims("annie", book.read) |
| 411 | c.now = func() time.Time { return clock } |
| 412 | var lines []string |
| 413 | c.logf = func(format string, a ...any) { lines = append(lines, fmt.Sprintf(format, a...)) } |
| 414 | for i := 0; i < 120; i++ { |
| 415 | c.decide(context.Background(), []string{"a.bab.test"}) |
| 416 | clock = clock.Add(edgesClaimTTL) |
| 417 | } |
| 418 | if len(lines) != 3 { // passes 1, 60 and 120 |
| 419 | t.Fatalf("%d lines for 120 failing passes, want 3: %q", len(lines), lines) |
| 420 | } |
| 421 | book.errs = nil |
| 422 | c.decide(context.Background(), []string{"a.bab.test"}) |
| 423 | clock = clock.Add(edgesClaimTTL) |
| 424 | c.decide(context.Background(), []string{"a.bab.test"}) |
| 425 | if len(lines) != 4 { |
| 426 | t.Fatalf("lines after recovery = %q, want one recovery line", lines) |
| 427 | } |
| 428 | } |
| 429 | |
amadan