| index 639479aba..7c8a229b9 100644 |
| --- a/internal/moves/readiness_test.go |
| +++ b/internal/moves/readiness_test.go |
| @@ -17,18 +17,41 @@ import ( |
| "github.com/carlosframework/platform/internal/release" |
| ) |
| |
| +// passingTarget is a passing report made at `at` with real evidence: a |
| +// 100 MiB database priced from rates measured a minute earlier, in a 120 s |
| +// window. goodSource is the matching source report. |
| func passingTarget(at time.Time) *TargetReport { |
| - r := &TargetReport{V: 1, At: at, Checks: map[string]Check{}, AttemptsAdmitted: 1} |
| + r := &TargetReport{V: 1, At: at, Checks: map[string]Check{}, CutBudgetS: 120} |
| for _, id := range AllChecks { |
| r.Checks[id] = Check{OK: true} |
| } |
| + m := Measurements{ |
| + Cross: MeasuredRate{MiBPerS: 10, At: at.Add(-time.Minute)}, |
| + S3: MeasuredRate{MiBPerS: 250, At: at.Add(-time.Minute)}, |
| + Disk: MeasuredRate{MiBPerS: 500, At: at.Add(-time.Minute)}, |
| + } |
| + res := BudgetCheck(100, m, 16<<20, 120, at) |
| + if !res.Check.OK { |
| + panic("the test fixture does not fit its own window: " + res.Check.Reason) |
| + } |
| + r.RecordBudget(100, m, res) |
| return r |
| } |
| |
| +func sourceWith(now time.Time, s1, s2 Check) *SourceReport { |
| + sr := goodSource(now) |
| + sr.S1, sr.S2 = s1, s2 |
| + return sr |
| +} |
| + |
| +func goodSource(now time.Time) *SourceReport { |
| + return &SourceReport{V: 1, At: now, S1: Check{OK: true}, S2: Check{OK: true}, CrossRate: 10, CrossRateAt: now.Add(-time.Minute)} |
| +} |
| + |
| func TestReadyNeedsFreshReports(t *testing.T) { |
| now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| pass := passingTarget |
| - src := &SourceReport{V: 1, At: now, S1: Check{OK: true}, S2: Check{OK: true}} |
| + src := goodSource(now) |
| if ok, why := Ready(1<<20, 16<<20, pass(now.Add(-time.Minute)), src, now); !ok { |
| t.Fatal(why) |
| } |
| @@ -37,7 +60,8 @@ func TestReadyNeedsFreshReports(t *testing.T) { |
| t.Fatalf("stale report promoted: %v %s", ok, why) |
| } |
| // The same bound on the source's report. |
| - oldSrc := &SourceReport{V: 1, At: now.Add(-11 * time.Minute), S1: Check{OK: true}, S2: Check{OK: true}} |
| + oldSrc := goodSource(now) |
| + oldSrc.At = now.Add(-11 * time.Minute) |
| if ok, why := Ready(0, 16<<20, pass(now), oldSrc, now); ok || !strings.Contains(why, "stale") { |
| t.Fatalf("stale source report promoted: %v %s", ok, why) |
| } |
| @@ -50,10 +74,10 @@ func TestReadyNeedsFreshReports(t *testing.T) { |
| if ok, why := Ready(0, 16<<20, bad, src, now); ok || !strings.Contains(why, "T11") { |
| t.Fatalf("failed check: %v %s", ok, why) |
| } |
| - if ok, _ := Ready(0, 16<<20, pass(now), &SourceReport{V: 1, At: now, S1: Check{Reason: "503"}, S2: Check{OK: true}}, now); ok { |
| + if ok, _ := Ready(0, 16<<20, pass(now), sourceWith(now, Check{Reason: "503"}, Check{OK: true}), now); ok { |
| t.Fatal("failed S1") |
| } |
| - if ok, why := Ready(0, 16<<20, pass(now), &SourceReport{V: 1, At: now, S1: Check{OK: true}, S2: Check{Reason: "strict spawn refused"}}, now); ok || !strings.Contains(why, "S2") { |
| + if ok, why := Ready(0, 16<<20, pass(now), sourceWith(now, Check{OK: true}, Check{Reason: "strict spawn refused"}), now); ok || !strings.Contains(why, "S2") { |
| t.Fatalf("failed S2: %v %s", ok, why) |
| } |
| if ok, _ := Ready(0, 16<<20, nil, src, now); ok { |
| @@ -77,7 +101,7 @@ func TestReadyNeedsFreshReports(t *testing.T) { |
| func TestStaleReportAfterReadyDemotesNothing(t *testing.T) { |
| now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| target := passingTarget(now) |
| - src := &SourceReport{V: 1, At: now, S1: Check{OK: true}, S2: Check{OK: true}} |
| + src := goodSource(now) |
| rec := &provision.Record{Move: &provision.MoveState{State: provision.MovePreparing}} |
| |
| if ok, why := Ready(0, 16<<20, target, src, now); !ok { |
| @@ -114,7 +138,8 @@ func TestReadyRejectsMismatchedInputs(t *testing.T) { |
| mk := func(targetFiles, sourceFiles, targetEnv, sourceEnv string) (*TargetReport, *SourceReport) { |
| tr := passingTarget(now) |
| tr.Inputs = provision.MoveInputs{FilesPlain: targetFiles, EnvReport: targetEnv} |
| - sr := &SourceReport{V: 1, At: now, S1: Check{OK: true}, S2: Check{OK: true}, FilesPlain: sourceFiles, Env: EnvReport{V: 1, Digest: sourceEnv}} |
| + sr := goodSource(now) |
| + sr.FilesPlain, sr.Env = sourceFiles, EnvReport{V: 1, Digest: sourceEnv} |
| return tr, sr |
| } |
| tr, sr := mk("sha256:aaa", "sha256:aaa", "env-1", "env-1") |
| @@ -134,28 +159,134 @@ func TestReadyRejectsMismatchedInputs(t *testing.T) { |
| } |
| |
| func TestReadyRefusesAnEstimateMadeUnderASmallerMirrorBar(t *testing.T) { |
| - now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| - src := &SourceReport{V: 1, At: now, S1: Check{OK: true}, S2: Check{OK: true}} |
| - tr := passingTarget(now) |
| - |
| - // Priced at the default 16 MiB bar (MirrorBar zero), the operator's |
| - // bar raised to 32 MiB: the estimate is too small for it. |
| - if ok, why := Ready(0, 32<<20, tr, src, now); ok || !strings.Contains(why, "T11") { |
| + now := t11Now |
| + src := goodSource(now) |
| + // made returns a report whose budget was priced for the given bar. |
| + made := func(barBytes int64) *TargetReport { |
| + tr := passingTarget(now) |
| + m := Measurements{ |
| + Cross: MeasuredRate{MiBPerS: 10, At: now.Add(-time.Minute)}, |
| + S3: MeasuredRate{MiBPerS: 250, At: tr.S3At}, |
| + Disk: MeasuredRate{MiBPerS: 500, At: tr.DiskAt}, |
| + } |
| + res := BudgetCheck(100, m, barBytes, 120, now) |
| + if !res.Check.OK { |
| + t.Fatalf("%+v", res) |
| + } |
| + tr.RecordBudget(100, m, res) |
| + return tr |
| + } |
| + // Priced at the 16 MiB default, the operator's bar raised to 32 MiB: |
| + // the estimate is too small for it. |
| + if ok, why := Ready(0, 32<<20, made(16<<20), src, now); ok || !strings.Contains(why, "T11") || !strings.Contains(why, "priced") { |
| t.Fatalf("under-priced estimate promoted: %v %q", ok, why) |
| } |
| - // Priced at 32 MiB, bar 32 MiB: fine. Priced at 32, bar lowered: fine. |
| - tr.Rates.MirrorBar = 32 |
| - if ok, why := Ready(0, 32<<20, tr, src, now); !ok { |
| + // Priced at 32 MiB: fine at 32, and fine at a lower bar. |
| + if ok, why := Ready(0, 32<<20, made(32<<20), src, now); !ok { |
| t.Fatal(why) |
| } |
| - if ok, why := Ready(0, 8<<20, tr, src, now); !ok { |
| + if ok, why := Ready(0, 8<<20, made(32<<20), src, now); !ok { |
| t.Fatal(why) |
| } |
| // Priced at 8, bar 16: too small. |
| - tr.Rates.MirrorBar = 8 |
| - if ok, _ := Ready(0, 16<<20, tr, src, now); ok { |
| + if ok, _ := Ready(0, 16<<20, made(8<<20), src, now); ok { |
| t.Fatal("a smaller priced bar promoted") |
| } |
| + // A mirror bar that is not finite never promotes, nor a negative one. |
| + for _, bar := range []float64{math.Inf(1), math.NaN(), -16} { |
| + tr := made(16 << 20) |
| + tr.Rates.MirrorBar = bar |
| + if ok, _ := Ready(0, 16<<20, tr, src, now); ok { |
| + t.Fatalf("mirror bar %v promoted", bar) |
| + } |
| + } |
| +} |
| + |
| +// The controller's fail-closed ruling: Ready re-checks the evidence behind |
| +// the budget at its own `now`, whatever the report's checks say. |
| +func TestReadyValidatesTheEvidenceItself(t *testing.T) { |
| + now := t11Now |
| + if ok, why := Ready(0, 16<<20, passingTarget(now), goodSource(now), now); !ok { |
| + t.Fatal(why) |
| + } |
| + bad := []float64{0, -1, math.NaN(), math.Inf(1), math.Inf(-1)} |
| + type c struct { |
| + name string |
| + mutate func(*TargetReport, *SourceReport) |
| + } |
| + var cases []c |
| + for _, v := range bad { |
| + v := v |
| + cases = append(cases, |
| + c{fmt.Sprintf("s3 rate %v", v), func(tr *TargetReport, _ *SourceReport) { tr.Rates.S3 = v }}, |
| + c{fmt.Sprintf("disk rate %v", v), func(tr *TargetReport, _ *SourceReport) { tr.Rates.Disk = v }}, |
| + c{fmt.Sprintf("cross rate in the source %v", v), func(_ *TargetReport, sr *SourceReport) { sr.CrossRate = v }}, |
| + c{fmt.Sprintf("cross rate in the target %v", v), func(tr *TargetReport, _ *SourceReport) { tr.Rates.Cross = v }}, |
| + ) |
| + } |
| + expired := now.Add(-MeasurementFreshness - time.Second) |
| + cases = append(cases, |
| + c{"s3 time missing", func(tr *TargetReport, _ *SourceReport) { tr.S3At = time.Time{} }}, |
| + c{"s3 expired", func(tr *TargetReport, _ *SourceReport) { tr.S3At = expired }}, |
| + c{"s3 from the future", func(tr *TargetReport, _ *SourceReport) { tr.S3At = now.Add(time.Hour) }}, |
| + c{"disk time missing", func(tr *TargetReport, _ *SourceReport) { tr.DiskAt = time.Time{} }}, |
| + c{"disk expired", func(tr *TargetReport, _ *SourceReport) { tr.DiskAt = expired }}, |
| + c{"disk from the future", func(tr *TargetReport, _ *SourceReport) { tr.DiskAt = now.Add(time.Hour) }}, |
| + c{"cross time missing", func(_ *TargetReport, sr *SourceReport) { sr.CrossRateAt = time.Time{} }}, |
| + c{"cross expired", func(_ *TargetReport, sr *SourceReport) { sr.CrossRateAt = expired }}, |
| + c{"cross from the future", func(_ *TargetReport, sr *SourceReport) { sr.CrossRateAt = now.Add(time.Hour) }}, |
| + c{"source has measured no cross rate", func(_ *TargetReport, sr *SourceReport) { sr.CrossRate, sr.CrossRateAt = 0, time.Time{} }}, |
| + c{"database size NaN", func(tr *TargetReport, _ *SourceReport) { tr.DBMiB = math.NaN() }}, |
| + c{"database size negative", func(tr *TargetReport, _ *SourceReport) { tr.DBMiB = -1 }}, |
| + c{"no window", func(tr *TargetReport, _ *SourceReport) { tr.CutBudgetS = 0 }}, |
| + c{"database too big for the window", func(tr *TargetReport, _ *SourceReport) { tr.DBMiB = 1e6 }}, |
| + c{"more attempts claimed than admitted", func(tr *TargetReport, _ *SourceReport) { tr.AttemptsAdmitted = ProbeAttemptsMax + 1 }}, |
| + c{"no attempts claimed", func(tr *TargetReport, _ *SourceReport) { tr.AttemptsAdmitted = 0 }}, |
| + ) |
| + for _, cs := range cases { |
| + tr, sr := passingTarget(now), goodSource(now) |
| + cs.mutate(tr, sr) |
| + ok, why := Ready(0, 16<<20, tr, sr, now) |
| + if ok || !strings.Contains(why, "T11") { |
| + t.Errorf("%s: ready=%v %q", cs.name, ok, why) |
| + } |
| + } |
| + |
| + // Which rate is named when a measurement is missing or expired. |
| + for _, w := range []struct { |
| + name string |
| + mutate func(*TargetReport, *SourceReport) |
| + which string |
| + }{ |
| + {"s3", func(tr *TargetReport, _ *SourceReport) { tr.S3At = expired }, "regional S3"}, |
| + {"disk", func(tr *TargetReport, _ *SourceReport) { tr.DiskAt = expired }, "the disk"}, |
| + {"cross", func(_ *TargetReport, sr *SourceReport) { sr.CrossRateAt = expired }, "the cross-region copy path"}, |
| + } { |
| + tr, sr := passingTarget(now), goodSource(now) |
| + w.mutate(tr, sr) |
| + if _, why := Ready(0, 16<<20, tr, sr, now); !strings.Contains(why, "no fresh rate measurement for "+w.which) { |
| + t.Errorf("%s: %q", w.name, why) |
| + } |
| + } |
| + |
| + // A measurement that expires while its report is still fresh is |
| + // refused: the target's report is 5 minutes old, its rates 58 minutes |
| + // old when written, so an hour later than their time they are stale. |
| + tr := passingTarget(now.Add(-5 * time.Minute)) |
| + tr.S3At, tr.DiskAt = now.Add(-58*time.Minute), now.Add(-58*time.Minute) |
| + if ok, why := Ready(0, 16<<20, tr, goodSource(now), now); !ok { |
| + t.Fatal(why) |
| + } |
| + if ok, why := Ready(0, 16<<20, tr, goodSource(now.Add(5*time.Minute)), now.Add(5*time.Minute)); ok || !strings.Contains(why, "T11") { |
| + t.Fatalf("a measurement that expired under a fresh report: %v %q", ok, why) |
| + } |
| + |
| + // A stored count at or below the recomputed one is accepted. |
| + tr = passingTarget(now) |
| + tr.AttemptsAdmitted = 1 |
| + if ok, why := Ready(0, 16<<20, tr, goodSource(now), now); !ok { |
| + t.Fatal(why) |
| + } |
| } |
| |
| func TestPassingNamesTheFirstFailureInIDOrder(t *testing.T) { |
| @@ -402,17 +533,16 @@ func TestT11BudgetCheckPricesTheOperatorsBar(t *testing.T) { |
| if zero.Seconds != def.Seconds { |
| t.Fatalf("%v vs %v", zero.Seconds, def.Seconds) |
| } |
| - // A report made by BudgetCheck is accepted by Ready at the same bar, |
| - // and refused at a larger one. |
| - src := &SourceReport{V: 1, At: t11Now, S1: Check{OK: true}, S2: Check{OK: true}} |
| + // A rejected budget never promotes: recording the 4 GiB result in a |
| + // report leaves it with zero attempts, and Ready refuses it. |
| + src := goodSource(t11Now) |
| tr := passingTarget(t11Now) |
| - tr.Rates, tr.AttemptsAdmitted = big.Rates, 1 |
| - if ok, why := Ready(0, 4<<30, tr, src, t11Now); !ok { |
| - t.Fatal(why) |
| + tr.RecordBudget(100, m, big) |
| + if tr.AttemptsAdmitted != 0 { |
| + t.Fatalf("%d", tr.AttemptsAdmitted) |
| } |
| - tr.Rates = def.Rates |
| if ok, _ := Ready(0, 4<<30, tr, src, t11Now); ok { |
| - t.Fatal("a report priced at 16 MiB promoted under a 4 GiB bar") |
| + t.Fatal("a report whose budget admits no attempt promoted") |
| } |
| } |
| |