| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "reflect" |
| 6 | "strings" |
| 7 | "testing" |
| 8 | |
| 9 | "github.com/carlosframework/platform/internal/hostclaim" |
| 10 | "github.com/carlosframework/platform/internal/park" |
| 11 | "github.com/carlosframework/platform/internal/provision" |
| 12 | ) |
| 13 | |
| 14 | // Moves into the home region, part 2, Task 4: the loss and the fenced |
| 15 | // release of a home target, from clean claim state. The fixtures are the |
| 16 | // loss world and Plan 13's fence world, each turned into a move into home: |
| 17 | // the record's To is the home's code "", the op's to is @home, written with |
| 18 | // home_region set to the target box's roster name, and the console names |
| 19 | // the same home. The home's roster name may appear only in what the claim |
| 20 | // recorded about the box itself (its admissions, the loss record's write |
| 21 | // sources); every region the moves code supplies is @home. |
| 22 | |
| 23 | // onlyHomeToken asserts a claim that started clean: every chore that is |
| 24 | // not the named source's is in @home, owned by region:@home; nothing names |
| 25 | // the home's roster name or the unset region in a chore, touched or |
| 26 | // quiet_after; and the op's to is still @home. |
| 27 | func onlyHomeToken(t *testing.T, c *hostclaim.Claim, source, rosterHome string) { |
| 28 | t.Helper() |
| 29 | for _, o := range c.Obligations { |
| 30 | switch { |
| 31 | case o.Kind == "park-local": |
| 32 | case o.Region == source: |
| 33 | if o.Owner != "region:"+source { |
| 34 | t.Fatalf("a source chore %+v", o) |
| 35 | } |
| 36 | case o.Region != park.HomeRegion || o.Owner != "region:"+park.HomeRegion: |
| 37 | t.Fatalf("a target chore not named %s: %+v", park.HomeRegion, o) |
| 38 | } |
| 39 | } |
| 40 | for _, r := range c.Touched.Regions { |
| 41 | if r == rosterHome || r == "" { |
| 42 | t.Fatalf("touched names %q: %+v", r, c.Touched) |
| 43 | } |
| 44 | } |
| 45 | for k := range c.QuietAfter { |
| 46 | if k == rosterHome || k == "" { |
| 47 | t.Fatalf("quiet_after names %q: %v", k, c.QuietAfter) |
| 48 | } |
| 49 | } |
| 50 | if to := c.Op.Intent["to"]; to != "" && to != park.HomeRegion { |
| 51 | t.Fatalf("the op's to is %q", to) |
| 52 | } |
| 53 | } |
| 54 | |
| 55 | // TestMoveIntoHomeDeclareLost: a home target (uk-box, roster region uk, |
| 56 | // the fleet's home) stopped mid-landing and declared lost through Plan 4 |
| 57 | // Task 10's door. The signed loss intent's TargetRegion is @home, copied |
| 58 | // from op.intent.to, and its SourceRegion is ie; ReleaseLostTarget succeeds |
| 59 | // with TargetRegion @home against intent.to @home and queues the target's |
| 60 | // location and lane parks in @home; the record cites the loss. The source |
| 61 | // is then re-admitted with the loss through AdmitWriterWithLoss and the |
| 62 | // moves loss owner, its predecessor the lost target's last admission in |
| 63 | // the spelling it recorded, uk. |
| 64 | func TestMoveIntoHomeDeclareLost(t *testing.T) { |
| 65 | ctx := context.Background() |
| 66 | w := newLossWorld(t, "landing", func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { |
| 67 | c.Op.Intent["to"], c.Op.Intent["home_region"] = park.HomeRegion, "uk" |
| 68 | r.Move.To, r.Region = "", "" |
| 69 | }) |
| 70 | w.control.homeRegion = "uk" |
| 71 | before := w.claim(t) |
| 72 | if len(before.Claim.Obligations) != 0 || len(before.Claim.QuietAfter) != 0 { |
| 73 | t.Fatalf("fixture: the claim is not clean: %+v", before.Claim) |
| 74 | } |
| 75 | onlyHomeToken(t, before.Claim, "ie", "uk") |
| 76 | |
| 77 | in := w.inspect(t, "uk-label") |
| 78 | if in.Details.Box != "uk-box" || in.Details.Op.Kind != hostclaim.KindLanding { |
| 79 | t.Fatalf("inspection = %+v", in.Details) |
| 80 | } |
| 81 | after, err := w.declare(w.submission(in)) |
| 82 | if err != nil { |
| 83 | t.Fatal(err) |
| 84 | } |
| 85 | id := in.Details.ID |
| 86 | loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, id) |
| 87 | if err != nil || loss.Record.State != "committed" { |
| 88 | t.Fatalf("loss record = %+v %v", loss.Record, err) |
| 89 | } |
| 90 | intent, _, err := lossIntentOf(loss.Record) |
| 91 | if err != nil { |
| 92 | t.Fatal(err) |
| 93 | } |
| 94 | if intent.TargetRegion != park.HomeRegion || intent.SourceRegion != "ie" || intent.Target != "uk-box" || intent.Source != "ie-box" { |
| 95 | t.Fatalf("the signed loss intent: %+v", intent) |
| 96 | } |
| 97 | // The write sources are what the claim recorded about the box: its own |
| 98 | // region, uk. |
| 99 | for _, s := range loss.Record.WriteSources { |
| 100 | if s.Reason == "target" && s.Region != "uk" { |
| 101 | t.Fatalf("a target write source %+v", s) |
| 102 | } |
| 103 | } |
| 104 | c := w.claim(t) |
| 105 | if c.ETag != after.ETag || c.Claim.Op.Kind != hostclaim.KindRestoring || c.Claim.Op.Phase != hostclaim.PhaseReleased || c.Claim.Op.Exec != "none" || |
| 106 | !hostclaim.LostTargetReleased(c.Claim, id) || c.Claim.Op.Intent["to"] != park.HomeRegion { |
| 107 | t.Fatalf("the claim after the declaration: %+v", c.Claim.Op) |
| 108 | } |
| 109 | parks := map[string]string{} |
| 110 | for _, o := range c.Claim.Obligations { |
| 111 | parks[o.Region+"|"+o.Park.Location] = o.Owner |
| 112 | } |
| 113 | if want := map[string]string{park.HomeRegion + "|instances/" + lossHost: "region:" + park.HomeRegion, park.HomeRegion + "|moves/" + lossHost + "/" + lossOp: "region:" + park.HomeRegion}; !reflect.DeepEqual(parks, want) { |
| 114 | t.Fatalf("parks = %v, want %v", parks, want) |
| 115 | } |
| 116 | onlyHomeToken(t, c.Claim, "ie", "uk") |
| 117 | lossRef, _ := hostclaim.LossRecordKey(id) |
| 118 | rec, _ := w.record(t) |
| 119 | if rb := rec.Move.Rollback; rec.Region != "ie" || rec.Move.To != "" || rb == nil || rb.LossRef != lossRef || rb.Reason != provision.RollbackDeclaredLost { |
| 120 | t.Fatalf("the record: region %q move %+v rollback %+v", rec.Region, rec.Move, rb) |
| 121 | } |
| 122 | |
| 123 | // The source takes execution back on the declaration's authority and |
| 124 | // probes; the recovery's admission carries the loss. |
| 125 | g := provision.MoveGuard{ID: lossOp, Incarnation: w.inc} |
| 126 | for _, step := range [][2]string{{provision.RbSourceWaiting, provision.RbSourceClaimed}, {provision.RbSourceClaimed, provision.RbSourceRestoring}, |
| 127 | {provision.RbSourceRestoring, provision.RbSourceVerified}, {provision.RbSourceVerified, provision.RbSourceProbing}} { |
| 128 | if _, err := provision.SetRecordMove(ctx, w.raw, "bab", "app", lossHost, provision.MoveRollbackSourcePhase(g, step[0], step[1], nil)); err != nil { |
| 129 | t.Fatalf("%s to %s: %v", step[0], step[1], err) |
| 130 | } |
| 131 | } |
| 132 | o := w.claim(t) |
| 133 | if o, err = hostclaim.Update(ctx, w.raw, o, hostclaim.ReturnToSource(lossOp, "ie-box", true, hostclaim.ReturnFence{})); err != nil { |
| 134 | t.Fatalf("the source's claimed: %v", err) |
| 135 | } |
| 136 | if o, err = hostclaim.Update(ctx, w.raw, o, hostclaim.SetMovePhase(lossOp, hostclaim.KindRestoring, hostclaim.PhaseClaimed, hostclaim.PhaseProbing)); err != nil { |
| 137 | t.Fatalf("the source's probing: %v", err) |
| 138 | } |
| 139 | h := o.Claim.Holder |
| 140 | last := h.Admissions[len(h.Admissions)-1] |
| 141 | if last.Box != "uk-box" || last.Region != "uk" { |
| 142 | t.Fatalf("fixture: the target's last admission %+v", last) |
| 143 | } |
| 144 | validated := make([]hostclaim.ValidatedWriteSource, 0, len(loss.Record.WriteSources)) |
| 145 | var more []hostclaim.WritesLost |
| 146 | for _, s := range loss.Record.WriteSources { |
| 147 | v := hostclaim.ValidatedWriteSource{Reason: s.Reason, Box: s.Box, Generation: s.Generation, Session: s.Session, Region: s.Region, Baseline: s.Baseline} |
| 148 | if s.Reason == "source-cut" { |
| 149 | v.RemoteMaxTXID = 42 |
| 150 | } |
| 151 | validated = append(validated, v) |
| 152 | if s.Box != last.Box || s.Generation != last.Generation || s.Session != last.Session { |
| 153 | more = append(more, hostclaim.WritesLost{Box: s.Box, Generation: s.Generation, Session: s.Session, RemoteMaxTXID: v.RemoteMaxTXID}) |
| 154 | } |
| 155 | } |
| 156 | admission := hostclaim.Admission{ObservedOpID: lossOp, ObservedPhase: hostclaim.PhaseProbing, ObservedSession: h.Session, ObservedGeneration: h.Generation, |
| 157 | Box: "ie-box", Mode: "cold", NewGeneration: "fresh1", Region: "ie", PredecessorRegion: last.Region, AdditionalWritesLost: more} |
| 158 | if err := MoveLossRecoveryOwner().ValidateLossRecovery(*o.Claim, loss.Record, admission); err != nil { |
| 159 | t.Fatalf("the moves loss owner on the signed intent: %v (claim %+v holder stamp %d)", err, o.Claim.Op, h.Stamp) |
| 160 | } |
| 161 | admitted, err := hostclaim.AdmitWriterWithLoss(ctx, w.raw, w.pub, o, admission, id, validated, MoveLossRecoveryOwner()) |
| 162 | if err != nil { |
| 163 | t.Fatalf("the source's admission with the loss: %v", err) |
| 164 | } |
| 165 | a := admitted.Claim.Holder.Admissions[len(admitted.Claim.Holder.Admissions)-1] |
| 166 | if a.LossID != id || a.Box != "ie-box" || a.Region != "ie" || a.Predecessor != last.Generation || a.PredecessorRegion != "uk" || a.WritesLost == nil || a.WritesLost.Box != "uk-box" { |
| 167 | t.Fatalf("the loss admission: %+v", a) |
| 168 | } |
| 169 | if !hostclaim.OperationLossConsumed(admitted.Claim, id) || admitted.Claim.Holder.WriterBox != "ie-box" { |
| 170 | t.Fatalf("the loss was not consumed: %+v", admitted.Claim.Holder) |
| 171 | } |
| 172 | onlyHomeToken(t, admitted.Claim, "ie", "uk") |
| 173 | |
| 174 | // The moves loss owner holds the loss intent's @home to op.intent.to |
| 175 | // exactly: a loss record whose intent names the roster name is refused. |
| 176 | bad := loss |
| 177 | named := intent |
| 178 | named.TargetRegion = "uk" |
| 179 | raw, err := named.encode() |
| 180 | if err != nil { |
| 181 | t.Fatal(err) |
| 182 | } |
| 183 | bad.Record.ConfirmedConsequences.RecoveryIntent = raw |
| 184 | if err := MoveLossRecoveryOwner().ValidateLossRecovery(*o.Claim, bad.Record, admission); err == nil { |
| 185 | t.Fatal("a loss intent naming uk was accepted against intent.to @home") |
| 186 | } |
| 187 | if err := MoveLossRecoveryOwner().ValidateLossRecovery(*o.Claim, loss.Record, admission); err != nil { |
| 188 | t.Fatalf("the signed intent: %v", err) |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | // TestMoveIntoHomeFencedTargetRelease: a home target terminated after it |
| 193 | // released, from clean claim state, on Plan 13's fence world. The recovery |
| 194 | // grant's ReleaseFencedTarget builds FencedTargetRelease with TargetRegion |
| 195 | // park.PlacementRegion(m.To), @home, and SourceRegion uk (the world's |
| 196 | // named source); hostclaim.ReleaseFencedTarget accepts it against |
| 197 | // intent.to @home; the location and lane parks it queues are in @home, |
| 198 | // owned by region:@home; the record keeps the home's code. |
| 199 | func TestMoveIntoHomeFencedTargetRelease(t *testing.T) { |
| 200 | w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { |
| 201 | c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindLanding, hostclaim.PhaseProbing, "source" |
| 202 | c.Op.Intent["source"], c.Op.Intent["target"], c.Op.Intent["target_enrollment"] = "target", "source", strings.Repeat("a", 32) |
| 203 | c.Op.Intent["to"], c.Op.Intent["home_region"] = park.HomeRegion, "ie" |
| 204 | c.Op.Participants = []string{"target", "source"} |
| 205 | c.Obligations = nil |
| 206 | c.Op.Journal = map[string]string{hostclaim.JournalSourceStamp: "1", hostclaim.JournalSrcGen: "src"} |
| 207 | r.Region = "uk" |
| 208 | r.Move.From, r.Move.To, r.Move.State = "uk", "", provision.MoveRollingBack |
| 209 | at := r.CreatedAt |
| 210 | key := "control/box-fences/" + strings.Repeat("a", 32) + ".json" |
| 211 | r.Move.Rollback = &provision.MoveRollback{ID: "rb_abcdefghijklmnopqrstuvwxyz", Reason: provision.RollbackTargetFenced, Evidence: key, RequestedAt: &at, TargetPhase: provision.RbTargetLost, LossRef: key, LostAt: &at, SourcePhase: provision.RbSourceWaiting} |
| 212 | }) |
| 213 | w.authority.home = "ie" |
| 214 | w.control.homeRegion = "ie" |
| 215 | before := w.claim(t).Claim |
| 216 | if len(before.Obligations) != 0 { |
| 217 | t.Fatalf("fixture: the claim owes %+v", before.Obligations) |
| 218 | } |
| 219 | onlyHomeToken(t, before, "uk", "ie") |
| 220 | run := func() error { |
| 221 | return w.control.RunRecovery(t.Context(), func(ctx context.Context) error { |
| 222 | return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error { |
| 223 | _, err := g.ReleaseFencedTarget(ctx, w.ref) |
| 224 | return err |
| 225 | }, w.authority) |
| 226 | }) |
| 227 | } |
| 228 | if err := run(); err != nil { |
| 229 | t.Fatal(err) |
| 230 | } |
| 231 | c := w.claim(t).Claim |
| 232 | if c.Op.Phase != hostclaim.PhaseReleased || c.Op.Intent["to"] != park.HomeRegion || c.Op.Journal["fence.source"] == "" { |
| 233 | t.Fatalf("the claim after the fenced release: %+v", c.Op) |
| 234 | } |
| 235 | parks := map[string]string{} |
| 236 | for _, o := range c.Obligations { |
| 237 | if o.Park == nil { |
| 238 | t.Fatalf("a chore without a scope: %+v", o) |
| 239 | } |
| 240 | parks[o.Region+"|"+o.Park.Location] = o.Owner |
| 241 | } |
| 242 | wantLocation, wantLane := false, false |
| 243 | for k, owner := range parks { |
| 244 | region, location, _ := strings.Cut(k, "|") |
| 245 | if region != park.HomeRegion || owner != "region:"+park.HomeRegion { |
| 246 | t.Fatalf("a park in %q owned by %q: %v", region, owner, parks) |
| 247 | } |
| 248 | wantLocation = wantLocation || location == strings.TrimSuffix(c.Holder.Location, "/") |
| 249 | wantLane = wantLane || strings.HasPrefix(location, "moves/"+w.host+"/") |
| 250 | } |
| 251 | if !wantLocation || !wantLane { |
| 252 | t.Fatalf("the fenced release's parks: %v", parks) |
| 253 | } |
| 254 | onlyHomeToken(t, c, "uk", "ie") |
| 255 | // The record's completion keeps the home's code. |
| 256 | if err := run(); err != nil { |
| 257 | t.Fatal("record completion", err) |
| 258 | } |
| 259 | rec, err := provision.GetRecord(t.Context(), w.raw, "bab", "app", w.host) |
| 260 | if err != nil || rec.Move.To != "" { |
| 261 | t.Fatalf("the record's target placement: %+v %v", rec.Move, err) |
| 262 | } |
| 263 | } |
| 264 | |
amadan