| 1 | package park |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "testing" |
| 7 | "time" |
| 8 | |
| 9 | "github.com/carlosframework/platform/internal/hostclaim" |
| 10 | "github.com/carlosframework/platform/internal/lease" |
| 11 | ) |
| 12 | |
| 13 | func TestPlan10MovesSweepRequiresPriorRetirement(t *testing.T) { |
| 14 | for _, tc := range []string{"missing", "zero", "equal", "later", "host", "etag", "move", "incarnation", "provider-error", "mutation", "valid"} { |
| 15 | t.Run(tc, func(t *testing.T) { |
| 16 | f := newFixture(t) |
| 17 | obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host) |
| 18 | if err != nil { |
| 19 | t.Fatal(err) |
| 20 | } |
| 21 | obs, err = hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error { |
| 22 | c.Obligations = nil |
| 23 | c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"} |
| 24 | return nil |
| 25 | }) |
| 26 | if err != nil { |
| 27 | t.Fatal(err) |
| 28 | } |
| 29 | f.b.Put("data", "moves/"+f.w.Host+"/old/staging", []byte("old")) |
| 30 | listing := f.e.Now() |
| 31 | listedAt := listing.Add(time.Second) |
| 32 | ticks := 0 |
| 33 | f.e.Now = func() time.Time { |
| 34 | ticks++ |
| 35 | if ticks == 1 { |
| 36 | return listing |
| 37 | } |
| 38 | return listedAt |
| 39 | } |
| 40 | f.e.View = func(_ context.Context, o hostclaim.Observed) (View, error) { |
| 41 | return View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "uk"}, nil |
| 42 | } |
| 43 | calls := 0 |
| 44 | if tc != "missing" { |
| 45 | f.e.VerifyMovesSweep = func(_ context.Context, o hostclaim.Observed, versions []lease.ObjectVersion) (MoveSweepProof, error) { |
| 46 | calls++ |
| 47 | if o.ETag != obs.ETag || len(versions) != 1 { |
| 48 | t.Fatal("proof not bound to claim and complete census") |
| 49 | } |
| 50 | p := MoveSweepProof{Host: o.Host, ClaimETag: o.ETag, MoveID: "old", Incarnation: f.ob.Park.Incarnation, RetiredAt: listing.Add(-time.Second)} |
| 51 | switch tc { |
| 52 | case "zero": |
| 53 | p.RetiredAt = time.Time{} |
| 54 | case "equal": |
| 55 | p.RetiredAt = listing |
| 56 | case "later": |
| 57 | p.RetiredAt = listing.Add(time.Second) |
| 58 | case "host": |
| 59 | p.Host = "other.example" |
| 60 | case "etag": |
| 61 | p.ClaimETag = "other" |
| 62 | case "move": |
| 63 | p.MoveID = "" |
| 64 | case "incarnation": |
| 65 | p.Incarnation = "wrong" |
| 66 | case "mutation": |
| 67 | o.Claim.Holder.Account = "foreign" |
| 68 | o.Claim.Lost = []string{"unrelated"} |
| 69 | case "provider-error": |
| 70 | return p, ErrPending |
| 71 | } |
| 72 | return p, nil |
| 73 | } |
| 74 | } |
| 75 | _, err = f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, f.ob.Park.RequestLimit) |
| 76 | fresh, readErr := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host) |
| 77 | if readErr != nil { |
| 78 | t.Fatal(readErr) |
| 79 | } |
| 80 | if tc != "valid" && tc != "mutation" { |
| 81 | if fresh.ETag != obs.ETag { |
| 82 | t.Fatalf("unsafe proof queued: err=%v claim=%+v", err, fresh.Claim) |
| 83 | } |
| 84 | return |
| 85 | } |
| 86 | if !errors.Is(err, ErrPending) || calls != 1 || len(fresh.Claim.Obligations) != 1 { |
| 87 | t.Fatalf("valid proof: calls=%d err=%v claim=%+v", calls, err, fresh.Claim) |
| 88 | } |
| 89 | ob := fresh.Claim.Obligations[0] |
| 90 | if ob.Park.Location != "moves/"+f.w.Host || !ob.Park.EvidenceAt.Equal(listedAt) { |
| 91 | t.Fatalf("scope does not bind listing time: %+v", ob) |
| 92 | } |
| 93 | if fresh.Claim.Holder.Account != obs.Claim.Holder.Account || len(fresh.Claim.Lost) != len(obs.Claim.Lost) { |
| 94 | t.Fatal("proof provider mutated CAS input") |
| 95 | } |
| 96 | // A retry keeps the queued bound; it does not restart the horizon. |
| 97 | f.e.Now = func() time.Time { return listing.Add(20 * time.Minute) } |
| 98 | if err := f.e.Run(t.Context(), Work{Host: f.w.Host, ID: ob.ID, Op: ob.Op}); err != nil { |
| 99 | t.Fatal(err) |
| 100 | } |
| 101 | }) |
| 102 | } |
| 103 | } |
| 104 | |
| 105 | func TestPlan10MovesSweepNoMoveOperation(t *testing.T) { |
| 106 | for _, kind := range []string{hostclaim.KindMoving, hostclaim.KindLanding, hostclaim.KindReverting, hostclaim.KindRestoring, hostclaim.KindRelocating} { |
| 107 | t.Run(kind, func(t *testing.T) { |
| 108 | p := MoveSweepProof{Host: "app.example", ClaimETag: "tag", MoveID: "old", Incarnation: "inc", RetiredAt: time.Unix(1, 0)} |
| 109 | o := hostclaim.Observed{Host: p.Host, ETag: p.ClaimETag, Claim: &hostclaim.Claim{Op: hostclaim.Op{Kind: kind, ID: "active"}}} |
| 110 | if err := p.Check(o, p.Incarnation, time.Unix(2, 0)); !errors.Is(err, ErrUnsafe) { |
| 111 | t.Fatalf("%s accepted: %v", kind, err) |
| 112 | } |
| 113 | }) |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | // Existing retirement tests advance the fake clock after listing so they can |
| 118 | // exercise completion. The precondition tests above keep the horizon pending. |
| 119 | func allowFixtureMoveSweep(f *fixture) { |
| 120 | listing := f.e.Now() |
| 121 | calls := 0 |
| 122 | f.e.Now = func() time.Time { |
| 123 | calls++ |
| 124 | if calls <= 2 { |
| 125 | return listing |
| 126 | } |
| 127 | return listing.Add(20 * time.Minute) |
| 128 | } |
| 129 | f.e.VerifyMovesSweep = func(_ context.Context, o hostclaim.Observed, _ []lease.ObjectVersion) (MoveSweepProof, error) { |
| 130 | return MoveSweepProof{Host: o.Host, ClaimETag: o.ETag, MoveID: "retired", Incarnation: f.ob.Park.Incarnation, RetiredAt: listing.Add(-time.Second)}, nil |
| 131 | } |
| 132 | } |
| 133 | |
| 134 | func TestPlan10MovesSweepPreservesInstanceEvidence(t *testing.T) { |
| 135 | f := newFixture(t) |
| 136 | f.seed() |
| 137 | o, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host) |
| 138 | if err != nil { |
| 139 | t.Fatal(err) |
| 140 | } |
| 141 | _, err = hostclaim.Update(t.Context(), f.e.Claims, o, func(c *hostclaim.Claim) error { c.Obligations = nil; return hostclaim.FreeAfterPark("op")(c) }) |
| 142 | if err != nil { |
| 143 | t.Fatal(err) |
| 144 | } |
| 145 | f.b.Put("data", "moves/"+f.w.Host+"/retired/staging", []byte("staged")) |
| 146 | listedAt := f.e.Now() |
| 147 | f.e.VerifyMovesSweep = func(_ context.Context, o hostclaim.Observed, _ []lease.ObjectVersion) (MoveSweepProof, error) { |
| 148 | return MoveSweepProof{Host: o.Host, ClaimETag: o.ETag, MoveID: "retired", Incarnation: f.ob.Park.Incarnation, RetiredAt: listedAt.Add(-time.Second)}, nil |
| 149 | } |
| 150 | scopes := map[string]time.Time{} |
| 151 | f.e.Authorize = func(_ context.Context, _ hostclaim.Observed, ob hostclaim.Obligation) error { |
| 152 | scopes[ob.Park.Location] = ob.Park.EvidenceAt |
| 153 | return nil |
| 154 | } |
| 155 | _, err = f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, f.ob.Park.RequestLimit) |
| 156 | if !errors.Is(err, ErrPending) { |
| 157 | t.Fatal(err) |
| 158 | } |
| 159 | if !scopes[f.ob.Park.Location].Equal(f.ob.Park.EvidenceAt) || !scopes["moves/"+f.w.Host].Equal(listedAt) { |
| 160 | t.Fatalf("evidence mixed: %+v", scopes) |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | func TestPlan10MovesSweepUnavailableProofKeepsOtherCandidates(t *testing.T) { |
| 165 | for _, unavailable := range []string{"nil", "error", "invalid"} { |
| 166 | t.Run(unavailable, func(t *testing.T) { |
| 167 | f := newFixture(t) |
| 168 | f.seed() |
| 169 | o, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host) |
| 170 | if err != nil { |
| 171 | t.Fatal(err) |
| 172 | } |
| 173 | _, err = hostclaim.Update(t.Context(), f.e.Claims, o, func(c *hostclaim.Claim) error { c.Obligations = nil; return hostclaim.FreeAfterPark("op")(c) }) |
| 174 | if err != nil { |
| 175 | t.Fatal(err) |
| 176 | } |
| 177 | f.b.Put("data", "moves/"+f.w.Host+"/retired/staging", []byte("staged")) |
| 178 | if unavailable != "nil" { |
| 179 | f.e.VerifyMovesSweep = func(context.Context, hostclaim.Observed, []lease.ObjectVersion) (MoveSweepProof, error) { |
| 180 | if unavailable == "error" { |
| 181 | return MoveSweepProof{}, ErrPending |
| 182 | } |
| 183 | return MoveSweepProof{}, nil |
| 184 | } |
| 185 | } |
| 186 | d, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.e.Now(), f.ob.Park.RequestLimit) |
| 187 | if !errors.Is(err, ErrPending) { |
| 188 | t.Fatalf("unrelated candidates not queued: %v", err) |
| 189 | } |
| 190 | if len(d.Candidates) == 0 { |
| 191 | t.Fatal("all candidates dropped") |
| 192 | } |
| 193 | fresh, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host) |
| 194 | if err != nil { |
| 195 | t.Fatal(err) |
| 196 | } |
| 197 | if len(fresh.Claim.Obligations) == 0 { |
| 198 | t.Fatal("nothing queued") |
| 199 | } |
| 200 | for _, ob := range fresh.Claim.Obligations { |
| 201 | if ob.Park.Location == "moves/"+f.w.Host { |
| 202 | t.Fatal("moves residue queued without proof") |
| 203 | } |
| 204 | } |
| 205 | }) |
| 206 | } |
| 207 | } |
| 208 | |
amadan