| 1 | package hostlife |
| 2 | |
| 3 | import "testing" |
| 4 | |
| 5 | // The readiness arithmetic of moves spec §7.3 T11 (rev 7.7; R3-7, R5-5, |
| 6 | // R6-3), in seconds. Rev 7.3's T11 required every one of three probe |
| 7 | // attempts, each with its settle, to fit the window beside the fixed costs: |
| 8 | // 150 s of probing alone, against a default window of 120 s, so no default |
| 9 | // move was ever eligible. Rev 7.4 made the permitted attempt count depend |
| 10 | // on the admitted budget. Rev 7.5 made the sealed transfer a full snapshot |
| 11 | // but still priced the final mirror at the 16 MiB bar (R5-5). Rev 7.6 |
| 12 | // built the snapshot on the target and priced three local passes, then |
| 13 | // restored the generation cold a fourth time without pricing it (R6-3). |
| 14 | // Rev 7.7 prices what the cut does, with regional S3 and the disk as |
| 15 | // separate rates: the final delta mirror within the bar across regions; |
| 16 | // on the target the restore of the staged replica (S3 read, disk write), |
| 17 | // the snapshot write and the anchor write (disk read, S3 write each), and |
| 18 | // the verification of the database the build produced (a disk read: its |
| 19 | // sha256 against the install manifest), which is what the `restored` |
| 20 | // phase does instead of a second cold restore. |
| 21 | // Rev 7.11 prices what the new protocol does at every probe attempt (R10-8): |
| 22 | // the admitted session purges the built database and restores the |
| 23 | // prepared generation again before the app starts (RestoreAttempt, one S3 |
| 24 | // read and one disk write of the whole database), plus the admission's |
| 25 | // own overhead (the CAS, the purge, generation.json: Admit); a retried |
| 26 | // probe is a fresh generation whose first upload is the whole database |
| 27 | // again (FreshUpload, one disk read and one S3 write), so every attempt |
| 28 | // after the first costs that too. |
| 29 | type cutEstimate struct { |
| 30 | Stop, Checkpoint, Mirror, Archive, Render, Open float64 |
| 31 | RestoreStaged, SnapshotBuild, Anchor, Verify float64 // target-side build, whole database |
| 32 | RestoreAttempt, Admit, FreshUpload float64 // per probe attempt (rev 7.11) |
| 33 | Probe, Settle float64 // one attempt |
| 34 | } |
| 35 | |
| 36 | const ( |
| 37 | cutWindowDefault = 120.0 // the operator's default cut window |
| 38 | probeAttemptsMax = 3 // §7.7: never more than three attempts |
| 39 | probeSeconds = 30.0 |
| 40 | settleSeconds = 20.0 |
| 41 | admitSeconds = 2.0 // the admitting CAS, the purge and generation.json (rev 7.11) |
| 42 | mirrorBarMiB = 16.0 // §7.3: the final round is at most this much |
| 43 | ) |
| 44 | |
| 45 | func (e cutEstimate) fixed() float64 { |
| 46 | return e.Stop + e.Checkpoint + e.Mirror + e.Archive + e.RestoreStaged + e.SnapshotBuild + e.Anchor + e.Verify + e.Render + e.Open |
| 47 | } |
| 48 | |
| 49 | // attempt prices the i-th probe attempt (from 1): the restore of the |
| 50 | // prepared generation and the admission overhead, the probe and its |
| 51 | // settle; from the second attempt on, the fresh generation's first upload |
| 52 | // as well (rev 7.11, R10-8). |
| 53 | func (e cutEstimate) attempt(i int) float64 { |
| 54 | c := e.RestoreAttempt + e.Admit + e.Probe + e.Settle |
| 55 | if i > 1 { |
| 56 | c += e.FreshUpload |
| 57 | } |
| 58 | return c |
| 59 | } |
| 60 | |
| 61 | // attemptsAdmitted is T11 as rev 7.4 states it, repriced in rev 7.11: the |
| 62 | // attempts permitted are those whose summed costs fit the window after |
| 63 | // the fixed costs, at most three; the instance is eligible when that is |
| 64 | // at least one, and the probe loop (§7.7) runs at most that many. |
| 65 | func (e cutEstimate) attemptsAdmitted(window float64) int { |
| 66 | left := window - e.fixed() |
| 67 | n := 0 |
| 68 | for n < probeAttemptsMax && left >= e.attempt(n+1) { |
| 69 | left -= e.attempt(n + 1) |
| 70 | n++ |
| 71 | } |
| 72 | return n |
| 73 | } |
| 74 | |
| 75 | // fitsRev73 is T11 as rev 7.3 stated it: three attempts, always. |
| 76 | func (e cutEstimate) fitsRev73(window float64) bool { |
| 77 | return e.fixed()+probeAttemptsMax*(e.Probe+e.Settle) <= window |
| 78 | } |
| 79 | |
| 80 | // rates are the measured rates T11 reads, in MiB/s: the cross-region copy |
| 81 | // path, regional S3 (reads and writes from a box in the bucket's region) |
| 82 | // and the box's disk. |
| 83 | type rates struct{ cross, s3, disk float64 } |
| 84 | |
| 85 | // estimate prices a cut for a database of dbMiB at the given rates, as |
| 86 | // rev 7.7 §7.3 does. Every target-side pass moves the whole database once |
| 87 | // across S3 and once across the disk, bar the verification, which reads |
| 88 | // the disk alone. |
| 89 | func estimate(dbMiB float64, r rates) cutEstimate { |
| 90 | return cutEstimate{ |
| 91 | Stop: 2, Checkpoint: 1, Archive: 2, Render: 1, Open: 5, |
| 92 | Mirror: mirrorBarMiB / r.cross, |
| 93 | RestoreStaged: dbMiB/r.s3 + dbMiB/r.disk, |
| 94 | SnapshotBuild: dbMiB/r.disk + dbMiB/r.s3, |
| 95 | Anchor: dbMiB/r.disk + dbMiB/r.s3, |
| 96 | Verify: dbMiB / r.disk, |
| 97 | // rev 7.11: the probe's admission restores the prepared generation |
| 98 | // again (purge, then the pinned snapshot from regional S3 to disk), |
| 99 | // with the admission's overhead; a retried probe uploads the whole |
| 100 | // database once more as its fresh generation's root |
| 101 | RestoreAttempt: dbMiB/r.s3 + dbMiB/r.disk, |
| 102 | Admit: admitSeconds, |
| 103 | FreshUpload: dbMiB/r.disk + dbMiB/r.s3, |
| 104 | Probe: probeSeconds, Settle: settleSeconds, |
| 105 | } |
| 106 | } |
| 107 | |
| 108 | // exampleRates are the rates the spec's worked example uses (§7.3 T11): |
| 109 | // 10 MiB/s across regions, 250 MiB/s to regional S3, 500 MiB/s on disk. |
| 110 | var exampleRates = rates{cross: 10, s3: 250, disk: 500} |
| 111 | |
| 112 | func TestReadinessBudget(t *testing.T) { |
| 113 | // a tiny instance: every fixed cost at the floor of its measured rate |
| 114 | tiny := estimate(1, exampleRates) |
| 115 | if tiny.fitsRev73(cutWindowDefault) { |
| 116 | t.Fatalf("rev 7.3's formula admits a tiny instance in the default window: R3-7 is stale") |
| 117 | } |
| 118 | if n := tiny.attemptsAdmitted(cutWindowDefault); n != 2 { |
| 119 | t.Fatalf("rev 7.4's formula admits %d attempts for a tiny instance in %.0f s, want 2", n, cutWindowDefault) |
| 120 | } |
| 121 | // the boundary: fixed costs of 70 s leave exactly one attempt; 71 s none |
| 122 | edge := cutEstimate{Stop: 70 - 5, Open: 5, Probe: probeSeconds, Settle: settleSeconds} |
| 123 | if n := edge.attemptsAdmitted(cutWindowDefault); n != 1 { |
| 124 | t.Fatalf("fixed costs of 70 s admit %d attempts, want 1", n) |
| 125 | } |
| 126 | edge.Stop++ |
| 127 | if n := edge.attemptsAdmitted(cutWindowDefault); n != 0 { |
| 128 | t.Fatalf("fixed costs of 71 s admit %d attempts, want 0 (refused)", n) |
| 129 | } |
| 130 | // a generous window never permits more than three |
| 131 | if n := tiny.attemptsAdmitted(600); n != probeAttemptsMax { |
| 132 | t.Fatalf("a 600 s window admits %d attempts, want %d", n, probeAttemptsMax) |
| 133 | } |
| 134 | // R5-5: the spec's example, a 2 GiB database at the example rates. Rev |
| 135 | // 7.5's full-snapshot transfer alone takes 205 s and could never fit; |
| 136 | // the delta within the bar and the target's build do, with one attempt |
| 137 | big := estimate(2048, exampleRates) |
| 138 | fullTransfer := 2048.0 / exampleRates.cross |
| 139 | if fullTransfer < cutWindowDefault { |
| 140 | t.Fatalf("the example no longer illustrates R5-5") |
| 141 | } |
| 142 | if n := big.attemptsAdmitted(cutWindowDefault); n != 1 { |
| 143 | t.Fatalf("a 2 GiB database at the example rates admits %d attempts, want 1 (fixed %.1f s, first attempt %.1f s)", n, big.fixed(), big.attempt(1)) |
| 144 | } |
| 145 | if big.fixed() < 53 || big.fixed() > 54 { |
| 146 | t.Fatalf("the example's fixed costs are %.1f s; the spec says 53.6 s", big.fixed()) |
| 147 | } |
| 148 | // rev 7.11 (R10-8): the first attempt restores the prepared generation |
| 149 | // again and pays the admission: 12.3 + 2 + 30 + 20 = 64.3 s, so the |
| 150 | // example runs to 117.9 s and admits one attempt by 2.1 s; a second |
| 151 | // attempt would add 76.6 s (a fresh generation's full upload too) |
| 152 | if a := big.attempt(1); a < 64 || a > 65 { |
| 153 | t.Fatalf("the example's first attempt costs %.1f s; the spec says 64.3 s", a) |
| 154 | } |
| 155 | if tot := big.fixed() + big.attempt(1); tot < 117 || tot > 118.5 { |
| 156 | t.Fatalf("the example runs to %.1f s with one attempt; the spec says 117.9 s", tot) |
| 157 | } |
| 158 | // the review's 2.5 GiB (R10-8): rev 7.10's arithmetic admitted it at |
| 159 | // 113.8 s; with the probe's own restore it is 129.2 s, past the window, |
| 160 | // and refused |
| 161 | bigger := estimate(2560, exampleRates) |
| 162 | old := bigger.fixed() + probeSeconds + settleSeconds |
| 163 | if old < 113 || old > 114.5 { |
| 164 | t.Fatalf("rev 7.10's arithmetic for 2.5 GiB is %.1f s; the review says 113.8 s", old) |
| 165 | } |
| 166 | if n := bigger.attemptsAdmitted(cutWindowDefault); n != 0 { |
| 167 | t.Fatalf("2.5 GiB at the example rates admits %d attempts, want 0 (%.1f s with one attempt)", n, bigger.fixed()+bigger.attempt(1)) |
| 168 | } |
| 169 | // and a cross-region snapshot transfer, if a builder did that instead, |
| 170 | // is refused by the same arithmetic |
| 171 | naive := big |
| 172 | naive.Mirror, naive.RestoreStaged, naive.SnapshotBuild = fullTransfer, 0, 0 |
| 173 | if n := naive.attemptsAdmitted(cutWindowDefault); n != 0 { |
| 174 | t.Fatalf("a full cross-region snapshot transfer admits %d attempts, want 0", n) |
| 175 | } |
| 176 | // R6-3: rev 7.6 priced three passes at one local rate and then restored |
| 177 | // the generation cold a fourth time, unpriced. The review's workload: 3 |
| 178 | // GiB, 200 MiB/s local, 10 MiB/s across regions, one probe. Three passes |
| 179 | // admit one attempt (108.68 s with the probe); the fourth pass takes the |
| 180 | // same workload to 124.04 s, past the window |
| 181 | r := rates{cross: 10, s3: 200, disk: 200} |
| 182 | three := estimate(3072, r) |
| 183 | three.RestoreStaged, three.SnapshotBuild, three.Anchor, three.Verify = 3072/200.0, 3072/200.0, 3072/200.0, 0 // rev 7.6's arithmetic |
| 184 | three.RestoreAttempt, three.Admit, three.FreshUpload = 0, 0, 0 // rev 7.6's arithmetic knew no per-attempt restore |
| 185 | if n := three.attemptsAdmitted(cutWindowDefault); n != 1 { |
| 186 | t.Fatalf("rev 7.6's three passes admit %d attempts for the review's workload, want 1 (the finding's premise; fixed %.2f s)", n, three.fixed()) |
| 187 | } |
| 188 | four := three |
| 189 | four.Verify = 3072 / 200.0 // the unpriced cold restore, at the same rate |
| 190 | if n := four.attemptsAdmitted(cutWindowDefault); n != 0 || four.fixed()+probeSeconds+settleSeconds < 124 { |
| 191 | t.Fatalf("with the fourth pass priced the review's workload admits %d attempts at %.2f s with one probe, want 0 past 124 s", n, four.fixed()+probeSeconds+settleSeconds) |
| 192 | } |
| 193 | // rev 7.7 verifies the built database instead of restoring it again: |
| 194 | // the same workload is refused honestly, by its own arithmetic |
| 195 | if n := estimate(3072, r).attemptsAdmitted(cutWindowDefault); n != 0 { |
| 196 | t.Fatalf("rev 7.7 admits %d attempts for the review's workload, want 0 (fixed %.1f s)", n, estimate(3072, r).fixed()) |
| 197 | } |
| 198 | } |
| 199 | |
amadan