amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostrelocate
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "strings"
8 "testing"
9 "time"
10
11 "github.com/carlosframework/platform/internal/hostclaim"
12)
13
14// releasedAndInstalling drives a drain to installing on dst, released.
15func releasedAndInstalling(t *testing.T, f *fixture) string {
16 t.Helper()
17 ctx := context.Background()
18 if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "src", Target: "dst"}); err != nil {
19 t.Fatal(err)
20 }
21 f.ack("src")
22 f.ack("dst")
23 f.worker(nil) // releasing
24 h := f.claim().Claim.Holder
25 id := f.claim().Claim.Op.ID
26 f.participant(hostclaim.JournalRelease(id, hostclaim.Release{ReleasedBy: "src", Enrollment: enrollment("src"), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
27 f.worker(nil) // pinning
28 f.worker(nil) // pin
29 f.worker(nil) // installing on dst
30 return id
31}
32
33// Astra round 3 finding 4, ruling 67: a passed probe whose stop receipt
34// never comes is bounded. ReleaseDeadline after the probe passed, the op
35// is stuck for an operator; before it, nothing changes; nothing is
36// released. Why the receipt never comes is the participant's (its two
37// scenarios are TestParticipantEndsNoProbeWithoutItsLease and
38// TestParticipantRecordsNoProbeStopWhileRootRefuses); the worker sees
39// only a probe with no probe_stop.
40func TestWorkerBoundsAPassedProbeAwaitingItsStop(t *testing.T) {
41 f := newFixture(t)
42 id := releasedAndInstalling(t, f)
43 f.installAndProbe("dst")
44 o := f.claim()
45 f.participant(hostclaim.RecordProbe(id, hostclaim.Probe{Box: "dst", Session: o.Claim.Holder.Session, At: f.now}))
46 f.now = f.now.Add(ReleaseDeadline)
47 if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" || o.Claim.Op.Kind != hostclaim.KindRelocating {
48 t.Fatalf("stuck before the bound: %+v", o.Claim.Op.Journal)
49 }
50 f.now = f.now.Add(time.Second)
51 o = f.worker(nil)
52 if o.Claim.Op.Journal["stuck"] == "" || o.Claim.Op.Kind != hostclaim.KindRelocating || o.Claim.Op.Exec != "dst" || o.Claim.Op.Journal["probe_stop"] != "" {
53 t.Fatalf("not stuck for an operator, or something released: %+v", o.Claim.Op)
54 }
55}
56
57// A probe journaled before rev 4 has no At: the bound counts from
58// probing_since plus ProbeDeadline.
59func TestWorkerBoundsAnUntimedPassedProbeFromProbingSince(t *testing.T) {
60 f := newFixture(t)
61 id := releasedAndInstalling(t, f)
62 f.installAndProbe("dst")
63 o := f.claim()
64 f.participant(hostclaim.RecordProbe(id, hostclaim.Probe{Box: "dst", Session: o.Claim.Holder.Session}))
65 f.now = f.now.Add(ProbeDeadline + ReleaseDeadline)
66 if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" {
67 t.Fatal("stuck before probing_since + ProbeDeadline + ReleaseDeadline")
68 }
69 f.now = f.now.Add(time.Second)
70 if o := f.worker(nil); o.Claim.Op.Journal["stuck"] == "" {
71 t.Fatalf("not stuck: %+v", o.Claim.Op)
72 }
73}
74
75// Ruling 60: an install that never enters probing (its stamp retired, its
76// participation denied) is stuck ProbeDeadline after installed_at.
77func TestWorkerBoundsAnInstallThatNeverProbes(t *testing.T) {
78 f := newFixture(t)
79 releasedAndInstalling(t, f)
80 f.worker(ErrNotYet) // P is not built yet
81 p := f.claim().Claim.Op.Journal["prepared"]
82 f.participant(func(c *hostclaim.Claim) error { c.Op.Journal["test.built"] = p; return nil })
83 if o := f.worker(nil); o.Claim.Op.Phase != hostclaim.PhaseInstalled || o.Claim.Op.Journal["installed_at"] == "" {
84 t.Fatalf("installed with its start: %s %+v", o.Claim.Op.Phase, o.Claim.Op.Journal)
85 }
86 f.now = f.now.Add(ProbeDeadline)
87 if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" {
88 t.Fatal("stuck before the bound")
89 }
90 f.now = f.now.Add(time.Second)
91 if o := f.worker(nil); o.Claim.Op.Journal["stuck"] == "" || o.Claim.Op.Phase != hostclaim.PhaseInstalled {
92 t.Fatalf("not stuck: %+v", o.Claim.Op)
93 }
94}
95
96// Ruling 60: a target that enters probing but never admits (its stamp
97// retired, root refusing) is probe-failed at the probe deadline, then
98// stuck ReleaseDeadline later for an operator. A live target is never
99// replaced automatically while claim ask C1 is open: exec stays dst
100// throughout.
101func TestWorkerBoundsARefusedProbe(t *testing.T) {
102 f := newFixture(t)
103 id := releasedAndInstalling(t, f)
104 p := f.claim().Claim.Op.Journal["prepared"]
105 f.worker(ErrNotYet) // P is not built yet
106 f.participant(func(c *hostclaim.Claim) error { c.Op.Journal["test.built"] = p; return nil })
107 f.worker(nil) // installed
108 var stamp uint64
109 if err := json.Unmarshal([]byte(f.claim().Claim.Op.Journal["install_stamp"]), &stamp); err != nil {
110 t.Fatal(err)
111 }
112 f.participant(hostclaim.RecordLocalInstall(id, hostclaim.LocalInstall{Box: "dst", Enrollment: enrollment("dst"), InstallStamp: stamp, FilesManifestVersion: 1, JournalSHA256: strings.Repeat("d", 64)}))
113 f.participant(hostclaim.ToProbing(id, f.now))
114 f.now = f.now.Add(ProbeDeadline)
115 if o := f.worker(ErrNotYet); o.Claim.Op.Phase != hostclaim.PhaseRelocatingProbing {
116 t.Fatalf("probe-failed before the probe deadline: %s", o.Claim.Op.Phase)
117 }
118 f.now = f.now.Add(time.Second)
119 if o := f.worker(nil); o.Claim.Op.Phase != hostclaim.PhaseProbeFailed || o.Claim.Op.Exec != "dst" {
120 t.Fatalf("not probe-failed on dst: %s %s", o.Claim.Op.Phase, o.Claim.Op.Exec)
121 }
122 f.now = f.now.Add(ReleaseDeadline)
123 if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" || o.Claim.Op.Exec != "dst" {
124 t.Fatalf("stuck or replaced before the release deadline: %+v", o.Claim.Op)
125 }
126 f.now = f.now.Add(time.Second)
127 if o := f.worker(nil); o.Claim.Op.Journal["stuck"] == "" || o.Claim.Op.Exec != "dst" || o.Claim.Op.Phase != hostclaim.PhaseProbeFailed {
128 t.Fatalf("not stuck on dst for an operator: %+v", o.Claim.Op)
129 }
130}
131
132// Fix round 1, M3: a timed probe's bound counts from Probe.At, not from
133// probing_since and not from probing_since plus ProbeDeadline. The probe
134// passes 7 minutes into probing: ReleaseDeadline after At the op waits,
135// a second later it is stuck.
136func TestWorkerBoundsATimedProbeFromItsPassNotProbingSince(t *testing.T) {
137 f := newFixture(t)
138 id := releasedAndInstalling(t, f)
139 f.installAndProbe("dst") // probing_since is f.now
140 passed := f.now.Add(7 * time.Minute)
141 o := f.claim()
142 f.participant(hostclaim.RecordProbe(id, hostclaim.Probe{Box: "dst", Session: o.Claim.Holder.Session, At: passed}))
143 f.now = passed.Add(ReleaseDeadline)
144 if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" {
145 t.Fatal("stuck before ReleaseDeadline after the probe passed: the bound counted from probing_since")
146 }
147 f.now = f.now.Add(time.Second)
148 if o := f.worker(nil); o.Claim.Op.Journal["stuck"] == "" {
149 t.Fatalf("not stuck ReleaseDeadline after the probe passed: %+v", o.Claim.Op)
150 }
151}
152
153// Fix round 1, M2: installed's bound needs its start. A missing or
154// unreadable installed_at is an error, never a pass with no bound; the op
155// is neither stuck nor waiting silently.
156func TestWorkerRefusesAnInstalledWithoutItsStart(t *testing.T) {
157 for name, value := range map[string]*string{"missing": nil, "unreadable": new("yesterday")} {
158 t.Run(name, func(t *testing.T) {
159 f := newFixture(t)
160 releasedAndInstalling(t, f)
161 f.worker(ErrNotYet) // P is not built yet
162 p := f.claim().Claim.Op.Journal["prepared"]
163 f.participant(func(c *hostclaim.Claim) error { c.Op.Journal["test.built"] = p; return nil })
164 f.worker(nil) // installed
165 f.participant(func(c *hostclaim.Claim) error {
166 if value == nil {
167 delete(c.Op.Journal, "installed_at")
168 } else {
169 c.Op.Journal["installed_at"] = *value
170 }
171 return nil
172 })
173 f.now = f.now.Add(ProbeDeadline + time.Hour)
174 o, err := f.w.Step(context.Background(), host)
175 if err == nil || errors.Is(err, ErrNotYet) {
176 t.Fatalf("an installed without its start: %v", err)
177 }
178 if o.Claim.Op.Journal["stuck"] != "" || f.claim().Claim.Op.Phase != hostclaim.PhaseInstalled {
179 t.Fatalf("the op moved: %+v", f.claim().Claim.Op)
180 }
181 })
182 }
183}
184
185// J3-3: a converted landing waits for an eligible target in starting, the
186// chooser's refusal returned each pass, and is stuck ReleaseDeadline after
187// its target's fence was observed, never pinned without a target.
188func TestWorkerBoundsAConversionWaitingForATarget(t *testing.T) {
189 f := newFixture(t)
190 converted := f.convertedLanding()
191 var fenced time.Time
192 for k, v := range converted.Claim.Op.Journal {
193 if strings.HasPrefix(k, "fence.") && strings.HasSuffix(k, ".observed_at") {
194 at, err := time.Parse(time.RFC3339Nano, v)
195 if err != nil {
196 t.Fatal(err)
197 }
198 fenced = at
199 }
200 }
201 if fenced.IsZero() {
202 t.Fatalf("the conversion journals no fence observation: %+v", converted.Claim.Op.Journal)
203 }
204 none := errors.New("no eligible box")
205 f.w.Choose = func(context.Context, hostclaim.Observed, []string) (hostclaim.Target, error) {
206 return hostclaim.Target{}, none
207 }
208 f.now = fenced.Add(ReleaseDeadline)
209 if o := f.worker(none); o.Claim.Op.Journal["stuck"] != "" || o.Claim.Op.Intent["target_box"] != "" {
210 t.Fatalf("stuck or named before the bound: %+v", o.Claim.Op)
211 }
212 f.now = f.now.Add(time.Second)
213 o := f.worker(nil)
214 if o.Claim.Op.Journal["stuck"] == "" || o.Claim.Op.Phase != hostclaim.PhaseStarting || o.Claim.Op.Journal["pin"] != "" {
215 t.Fatalf("not stuck for an operator: %+v", o.Claim.Op)
216 }
217}
218
219// Absence is never evidence: a converted landing whose fence observation
220// does not parse, or is gone, has nothing to bound its wait. The chooser's
221// refusal comes back with that defect named (ErrRefused), long after the
222// bound would have passed, and nothing is written: it is neither waited on
223// silently for ever nor stuck on a guess.
224func TestWorkerRefusesAConversionWaitWithNoReadableFenceObservation(t *testing.T) {
225 for name, change := range map[string]func(map[string]string){
226 "unparseable": func(j map[string]string) {
227 for k := range j {
228 if strings.HasPrefix(k, "fence.") && strings.HasSuffix(k, ".observed_at") {
229 j[k] = "not a time"
230 }
231 }
232 },
233 "unparseable beside a readable one": func(j map[string]string) {
234 for k := range j {
235 if strings.HasPrefix(k, "fence.") && strings.HasSuffix(k, ".observed_at") {
236 j[k] = "not a time"
237 }
238 }
239 j["fence.spa.observed_at"] = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339Nano)
240 },
241 "missing": func(j map[string]string) {
242 for k := range j {
243 if strings.HasPrefix(k, "fence.") && strings.HasSuffix(k, ".observed_at") {
244 delete(j, k)
245 }
246 }
247 },
248 } {
249 t.Run(name, func(t *testing.T) {
250 f := newFixture(t)
251 converted := f.step(f.convertedLanding(), func(c *hostclaim.Claim) error { change(c.Op.Journal); return nil })
252 none := errors.New("no eligible box")
253 f.w.Choose = func(context.Context, hostclaim.Observed, []string) (hostclaim.Target, error) {
254 return hostclaim.Target{}, none
255 }
256 f.now = f.now.Add(10 * ReleaseDeadline)
257 o, err := f.w.Step(context.Background(), host)
258 if !errors.Is(err, none) || !errors.Is(err, ErrRefused) {
259 t.Fatalf("step: %v, want the chooser's refusal and the defect", err)
260 }
261 if o.ETag != converted.ETag || f.claim().ETag != converted.ETag {
262 t.Fatalf("the op was written: %+v", f.claim().Claim.Op)
263 }
264 })
265 }
266}
267