| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "database/sql" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "github.com/carlosframework/platform/internal/hostbox" |
| 10 | "github.com/carlosframework/platform/internal/hostclaim" |
| 11 | "github.com/carlosframework/platform/internal/lease" |
| 12 | "github.com/carlosframework/platform/internal/moves" |
| 13 | "github.com/carlosframework/platform/internal/park" |
| 14 | "github.com/carlosframework/platform/internal/registry" |
| 15 | "path/filepath" |
| 16 | "reflect" |
| 17 | "testing" |
| 18 | "time" |
| 19 | ) |
| 20 | |
| 21 | func TestPlan12InventoryReplayChecksAllTimers(t *testing.T) { |
| 22 | t.Parallel() |
| 23 | for _, mode := range []string{"foreign-owner", "stale-incarnation", "route-changed", "ok"} { |
| 24 | t.Run(mode, func(t *testing.T) { |
| 25 | f, prior, h := plan12Handover(t, hostclaim.KindRestoring, "stop") |
| 26 | path := filepath.Join(t.TempDir(), "registry.db") |
| 27 | original, err := h.Registry.Get(prior.Host) |
| 28 | if err != nil { |
| 29 | t.Fatal(err) |
| 30 | } |
| 31 | h.Registry, err = registry.Open(path) |
| 32 | if err != nil { |
| 33 | t.Fatal(err) |
| 34 | } |
| 35 | defer h.Registry.Close() |
| 36 | if err = h.Registry.Upsert(*original); err != nil { |
| 37 | t.Fatal(err) |
| 38 | } |
| 39 | in := hostclaim.Admission{ObservedOpID: prior.Claim.Op.ID, ObservedPhase: prior.Claim.Op.Phase, ObservedSession: prior.Claim.Holder.Session, ObservedGeneration: prior.Claim.Holder.Generation, Box: h.Box, Mode: "cold", NewGeneration: "successor", Region: "ie", PredecessorRegion: "uk"} |
| 40 | admitted, err := hostclaim.Update(t.Context(), f.claims, prior, hostclaim.AdmitWriter(in)) |
| 41 | if err != nil { |
| 42 | t.Fatal(err) |
| 43 | } |
| 44 | if err := h.After(t.Context(), prior, admitted); err != nil { |
| 45 | t.Fatal(err) |
| 46 | } |
| 47 | row, err := h.Registry.Get(prior.Host) |
| 48 | if err != nil { |
| 49 | t.Fatal(err) |
| 50 | } |
| 51 | if mode == "route-changed" { |
| 52 | row.Version = "changed" |
| 53 | if err := h.Registry.Upsert(*row); err != nil { |
| 54 | t.Fatal(err) |
| 55 | } |
| 56 | } |
| 57 | if mode == "foreign-owner" || mode == "stale-incarnation" { |
| 58 | owner := row.ClaimOwnership |
| 59 | owner.ClaimRowID = "injected" |
| 60 | if mode == "foreign-owner" { |
| 61 | owner.ClaimOwner = "another-box" |
| 62 | } else { |
| 63 | owner.ClaimIncarnation = "stale" |
| 64 | } |
| 65 | if err := h.Registry.UpsertTimer(registry.Timer{Host: prior.Host, Name: "injected", Kind: "cron", Spec: "* * * * *"}); err != nil { |
| 66 | t.Fatal(err) |
| 67 | } |
| 68 | raw, err := sql.Open("sqlite", path) |
| 69 | if err != nil { |
| 70 | t.Fatal(err) |
| 71 | } |
| 72 | defer raw.Close() |
| 73 | if _, err := raw.Exec("UPDATE timers SET claim_incarnation=?,claim_stamp=?,claim_owner=?,claim_row_id=? WHERE host=?", owner.ClaimIncarnation, owner.ClaimStamp, owner.ClaimOwner, owner.ClaimRowID, prior.Host); err != nil { |
| 74 | t.Fatal(err) |
| 75 | } |
| 76 | } |
| 77 | err = h.After(t.Context(), prior, admitted) |
| 78 | if (err == nil) != (mode == "ok") { |
| 79 | t.Fatalf("replay %s: %v", mode, err) |
| 80 | } |
| 81 | }) |
| 82 | } |
| 83 | } |
| 84 | |
| 85 | func TestPlan12InitialSourceRecoveryRequiresRootStop(t *testing.T) { |
| 86 | t.Parallel() |
| 87 | for _, session := range []uint64{0, 2} { |
| 88 | t.Run(fmt.Sprint(session), func(t *testing.T) { |
| 89 | f, o, _ := plan12Handover(t, hostclaim.KindRestoring, "stop") |
| 90 | key := moves.StoppedKey(moves.ReportPrefix("bab", "jam", o.Host, o.Claim.Op.ID), "source") |
| 91 | report, _ := json.Marshal(plan12Stopped(o, "source", "box-1")) |
| 92 | if _, err := f.claims.Put(t.Context(), key, report, ""); err != nil { |
| 93 | t.Fatal(err) |
| 94 | } |
| 95 | o, err := hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error { |
| 96 | c.Holder.Session = session |
| 97 | c.Holder.WriterBox, c.Holder.Generation = "box-1", "src" |
| 98 | c.Holder.Admissions = []hostclaim.AdmissionRecord{{Session: session, Box: "box-1", Generation: "src", Mode: "cold", Region: "ie"}} |
| 99 | if session == 0 { |
| 100 | c.Holder.WriterBox, c.Holder.Generation = "", "" |
| 101 | c.Holder.Admissions = nil |
| 102 | } |
| 103 | c.Op.Journal[hostclaim.JournalCutSession] = fmt.Sprint(session) |
| 104 | c.Op.Journal[hostclaim.JournalSrcGen] = c.Holder.Generation |
| 105 | c.Op.Journal[hostclaim.JournalEvidenceSource] = key |
| 106 | return nil |
| 107 | }) |
| 108 | if err != nil { |
| 109 | t.Fatal(err) |
| 110 | } |
| 111 | c := f.controller("boot-1", "controller-1", 4101, "start-1") |
| 112 | c.wake.Continue = hostclaim.SessionContinues |
| 113 | c.assembly.Guard = f.guard |
| 114 | c.assembly.bindClaimWakeStop() |
| 115 | if err := c.wake.Stop(t.Context(), o); err == nil { |
| 116 | t.Fatal("agent report substituted for root stop") |
| 117 | } |
| 118 | if len(f.requests) == 0 { |
| 119 | t.Fatal("root was not asked to stop the cut installation") |
| 120 | } |
| 121 | }) |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | type plan12RetirementEvidence struct { |
| 126 | parkRegistrationEvidence |
| 127 | seen []lease.ObjectVersion |
| 128 | at time.Time |
| 129 | err error |
| 130 | } |
| 131 | |
| 132 | func (e *plan12RetirementEvidence) VerifyRetirement(_ context.Context, o hostclaim.Observed, v []lease.ObjectVersion) (string, time.Time, error) { |
| 133 | e.seen = append([]lease.ObjectVersion(nil), v...) |
| 134 | return o.Claim.Holder.Incarnation, e.at, e.err |
| 135 | } |
| 136 | func TestPlan12RetirementUsesOriginalEvidenceAndFullCensus(t *testing.T) { |
| 137 | for _, refuses := range []bool{false, true} { |
| 138 | t.Run(fmt.Sprint(refuses), func(t *testing.T) { |
| 139 | f := newRecoveryFixture(t) |
| 140 | o := f.read() |
| 141 | versions := []lease.ObjectVersion{{Key: o.Claim.Holder.Location + "/g-old/db", VersionID: "one"}, {Key: "moves/" + o.Host + "/old/db", VersionID: "two"}} |
| 142 | proof := &plan12RetirementEvidence{at: time.Unix(100, 0)} |
| 143 | if refuses { |
| 144 | proof.err = errors.New("move did not retire this generation") |
| 145 | } |
| 146 | p := productionClaimRegional{Now: func() time.Time { return time.Unix(200, 0) }, Parks: &claimMoveParkSources{Evidence: proof}} |
| 147 | _, at, err := p.retirement(t.Context(), o, versions) |
| 148 | if !reflect.DeepEqual(proof.seen, versions) { |
| 149 | t.Fatalf("incomplete retirement census: %+v", proof.seen) |
| 150 | } |
| 151 | if refuses { |
| 152 | if !errors.Is(err, proof.err) { |
| 153 | t.Fatalf("ignored refusal: %v", err) |
| 154 | } |
| 155 | } else if err != nil || !at.Equal(proof.at) { |
| 156 | t.Fatalf("retirement time=%v error=%v", at, err) |
| 157 | } |
| 158 | }) |
| 159 | } |
| 160 | } |
| 161 | func TestPlan12RetirementRefusalAlertOncePerLocation(t *testing.T) { |
| 162 | o := plan12Retired(t) |
| 163 | loc := "instances/" + o.Host |
| 164 | view := park.View{ClaimETag: o.ETag, Region: "ie", RefusedLocations: map[string]error{loc: claimRetirementDependency()}} |
| 165 | versions := []lease.ObjectVersion{{Key: loc + "/0000/db", VersionID: "one"}, {Key: loc + "/0000/db", VersionID: "two"}} |
| 166 | d, err := park.Decide(o, view, versions, "") |
| 167 | if err != nil || len(d.Alerts) != 1 || len(d.Candidates) != 0 { |
| 168 | t.Fatalf("decision=%+v err=%v", d, err) |
| 169 | } |
| 170 | } |
| 171 | |
| 172 | func TestPlan12PendingRecoveryRejectsChangedIdentity(t *testing.T) { |
| 173 | t.Parallel() |
| 174 | for _, field := range []string{"same", "session", "op", "writer", "incarnation", "stamp", "generation", "kind", "phase", "exec"} { |
| 175 | t.Run(field, func(t *testing.T) { |
| 176 | f, o, h := plan12Handover(t, hostclaim.KindRestoring, "stop") |
| 177 | h.Launches = f.journal |
| 178 | in := hostclaim.Admission{ObservedOpID: o.Claim.Op.ID, ObservedPhase: o.Claim.Op.Phase, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: h.Box, Mode: "cold", NewGeneration: "successor", Region: "ie", PredecessorRegion: "uk"} |
| 179 | in, err := h.prepareRecovery(t.Context(), o, in) |
| 180 | if err != nil { |
| 181 | t.Fatal(err) |
| 182 | } |
| 183 | next, err := hostclaim.Update(t.Context(), f.claims, o, hostclaim.AdmitWriter(in)) |
| 184 | if err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | c := next.Claim |
| 188 | holder := c.Holder |
| 189 | switch field { |
| 190 | case "session": |
| 191 | holder.Session++ |
| 192 | holder.Stamp = holder.Session |
| 193 | holder.Admissions[len(holder.Admissions)-1].Session = holder.Session |
| 194 | case "op": |
| 195 | c.Op.ID = "other" |
| 196 | case "writer": |
| 197 | holder.WriterBox = "other" |
| 198 | case "incarnation": |
| 199 | holder.Incarnation = "other" |
| 200 | case "stamp": |
| 201 | holder.Stamp++ |
| 202 | case "generation": |
| 203 | holder.Generation = "other" |
| 204 | case "kind": |
| 205 | c.Op.Kind = hostclaim.KindReverting |
| 206 | case "phase": |
| 207 | c.Op.Phase = "other" |
| 208 | case "exec": |
| 209 | c.Op.Exec = "other" |
| 210 | } |
| 211 | _, pending, err := h.pendingRecovery(t.Context(), next) |
| 212 | if field == "same" { |
| 213 | if err != nil || !pending { |
| 214 | t.Fatal(pending, err) |
| 215 | } |
| 216 | } else if !errors.Is(err, hostbox.ErrChanged) { |
| 217 | t.Fatalf("changed %s accepted: %v", field, err) |
| 218 | } |
| 219 | }) |
| 220 | } |
| 221 | } |
| 222 | |
amadan