| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "reflect" |
| 7 | "testing" |
| 8 | "time" |
| 9 | |
| 10 | "github.com/carlosframework/platform/internal/hostclaim" |
| 11 | "github.com/carlosframework/platform/internal/provision" |
| 12 | ) |
| 13 | |
| 14 | func TestJ4DestroyedLocalOnlyAfterCommit(t *testing.T) { |
| 15 | w := newLossWorld(t, "landing", func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { |
| 16 | c.Op.Phase = hostclaim.PhaseCommitted |
| 17 | r.Move.State = provision.MoveCommitted |
| 18 | c.Obligations = []hostclaim.Obligation{ |
| 19 | {ID: "local", Op: lossOp, Kind: "park-local", Box: "ie-box", Retirement: "ret-local", Owner: "box:ie-box"}, |
| 20 | {ID: "remote", Op: lossOp, Kind: "park-replica", Region: "ie", Retirement: "ret-remote", Owner: "box:ie-box", Park: &hostclaim.ParkScope{Location: c.Holder.Location, Incarnation: c.Holder.Incarnation, EvidenceAt: r.CreatedAt, RequestLimit: 1}}, |
| 21 | {ID: "other", Op: "older-op", Kind: "park-local", Box: "uk-box", Retirement: "ret-other", Owner: "box:uk-box"}, |
| 22 | } |
| 23 | }) |
| 24 | before := w.claim(t) |
| 25 | record, tag := w.record(t) |
| 26 | in := w.inspect(t, "ie-label") |
| 27 | if in.Details.Consequences.Transition != "destroyed-local-only" || len(in.Details.Consequences.DischargeLocal) != 1 || in.Details.Consequences.PossibleWriteLoss || in.Details.Consequences.PossibleTimerLoss { |
| 28 | t.Fatalf("effects %+v", in.Details.Consequences) |
| 29 | } |
| 30 | sub := w.submission(in) |
| 31 | sub.Observation.Region = "eu-west-1" |
| 32 | sub.Observation.InstanceID = "i-ie" |
| 33 | got, err := w.control.DeclareMoveLoss(context.Background(), w.signer, lossBucket, "operator", "bab", "app", lossHost, "ie-box", sub) |
| 34 | if err != nil { |
| 35 | t.Fatal(err) |
| 36 | } |
| 37 | if !reflect.DeepEqual(before.Claim.Op, got.Claim.Op) || !reflect.DeepEqual(before.Claim.Holder, got.Claim.Holder) || !reflect.DeepEqual(before.Claim.Inventory, got.Claim.Inventory) { |
| 38 | t.Fatal("local discharge changed execution, sessions, holder or inventory") |
| 39 | } |
| 40 | if !reflect.DeepEqual(got.Claim.Obligations, before.Claim.Obligations[1:]) { |
| 41 | t.Fatalf("other obligations changed: %+v", got.Claim.Obligations) |
| 42 | } |
| 43 | after, afterTag := w.record(t) |
| 44 | if tag != afterTag || !reflect.DeepEqual(record, after) { |
| 45 | t.Fatal("local discharge wrote committed record") |
| 46 | } |
| 47 | if _, err := w.control.DeclareMoveLoss(context.Background(), w.signer, lossBucket, "operator", "bab", "app", lossHost, "ie-box", sub); err != nil { |
| 48 | t.Fatal("replay", err) |
| 49 | } |
| 50 | // A local-only signed record grants no second claim or record write. |
| 51 | scoped := &lossScopedStore{raw: w.raw, pub: w.pub, now: func() time.Time { return w.clock }, b: lossBinding{lossID: in.Details.ID, host: lossHost, opID: lossOp, kind: hostclaim.KindLanding, box: "ie-box", claimETag: before.ETag, incarnation: w.inc, account: "bab", app: "app"}} |
| 52 | for _, key := range []string{w.key, w.claimK} { |
| 53 | b, tag, err := w.raw.Get(context.Background(), key) |
| 54 | if err != nil { |
| 55 | t.Fatal(err) |
| 56 | } |
| 57 | if _, err := scoped.Put(context.Background(), key, b, tag); err == nil { |
| 58 | t.Fatal("local-only grant authorized follow-up write", key) |
| 59 | } |
| 60 | } |
| 61 | } |
| 62 | |
| 63 | func TestJ4DestroyedLocalOnlyAlreadyDeclaredTarget(t *testing.T) { |
| 64 | w := newLossWorld(t, "landing", func(c *hostclaim.Claim, _ *provision.Record, _ *AppAdmission) { |
| 65 | c.Inventory = map[string]hostclaim.InventoryEntry{"uk-box": {Paths: map[string]uint64{"root": 21}}} |
| 66 | }) |
| 67 | if _, err := w.declare(w.submission(w.inspect(t, "uk-label"))); err != nil { |
| 68 | t.Fatal(err) |
| 69 | } |
| 70 | before := w.claim(t) |
| 71 | _, tag := w.record(t) |
| 72 | var local hostclaim.Obligation |
| 73 | for _, o := range before.Claim.Obligations { |
| 74 | if o.Kind == "park-local" && o.Box == "uk-box" { |
| 75 | local = o |
| 76 | } |
| 77 | } |
| 78 | if local.ID == "" { |
| 79 | t.Fatal("lost target installation has no park-local obligation") |
| 80 | } |
| 81 | in := w.inspect(t, "uk-label") |
| 82 | sub := w.submission(in) |
| 83 | got, err := w.declare(sub) |
| 84 | if err != nil { |
| 85 | t.Fatal(err) |
| 86 | } |
| 87 | if !reflect.DeepEqual(before.Claim.Op, got.Claim.Op) || !reflect.DeepEqual(before.Claim.Holder, got.Claim.Holder) { |
| 88 | t.Fatal("second declaration changed data authority") |
| 89 | } |
| 90 | want := *before.Claim |
| 91 | want.Obligations = append([]hostclaim.Obligation(nil), before.Claim.Obligations...) |
| 92 | if err := hostclaim.CompleteObligation(local.Op, local.ID)(&want); err != nil { |
| 93 | t.Fatal(err) |
| 94 | } |
| 95 | if !reflect.DeepEqual(want.Obligations, got.Claim.Obligations) { |
| 96 | t.Fatal("regional or staging obligations changed") |
| 97 | } |
| 98 | _, afterTag := w.record(t) |
| 99 | if tag != afterTag { |
| 100 | t.Fatal("local-only changed rollback record") |
| 101 | } |
| 102 | loss, err := hostclaim.ReadLossRecord(context.Background(), w.raw, w.pub, in.Details.ID) |
| 103 | if err != nil { |
| 104 | t.Fatal(err) |
| 105 | } |
| 106 | if len(loss.Record.WriteSources) != 0 || loss.Record.ConfirmedConsequences.RevokeExecution || loss.Record.ConfirmedConsequences.RemoveParticipation { |
| 107 | t.Fatal("local-only reopened data authority") |
| 108 | } |
| 109 | } |
| 110 | |
| 111 | func TestJ4DestroyedLocalOnlyKeepsOperationLossGate(t *testing.T) { |
| 112 | w := newLossWorld(t, "landing", func(c *hostclaim.Claim, _ *provision.Record, _ *AppAdmission) { |
| 113 | c.Inventory = map[string]hostclaim.InventoryEntry{"uk-box": {Paths: map[string]uint64{"root": 21}}} |
| 114 | }) |
| 115 | first := w.inspect(t, "uk-label") |
| 116 | before, err := w.declare(w.submission(first)) |
| 117 | if err != nil { |
| 118 | t.Fatal(err) |
| 119 | } |
| 120 | if id, err := hostclaim.OperationLossID(before.Claim); err != nil || id != first.Details.ID { |
| 121 | t.Fatalf("target loss gate = %q, %v", id, err) |
| 122 | } |
| 123 | local := w.inspect(t, "uk-label") |
| 124 | sub := w.submission(local) |
| 125 | for _, step := range []string{"discharge", "replay"} { |
| 126 | got, err := w.declare(sub) |
| 127 | if err != nil { |
| 128 | t.Fatal(step, err) |
| 129 | } |
| 130 | if id, err := hostclaim.OperationLossID(got.Claim); err != nil || id != first.Details.ID { |
| 131 | t.Fatalf("%s changed target loss gate: %q, %v", step, id, err) |
| 132 | } |
| 133 | if !reflect.DeepEqual(before.Claim.Lost, got.Claim.Lost) || !reflect.DeepEqual(before.Claim.Op, got.Claim.Op) { |
| 134 | t.Fatal(step, "added loss or changed operation") |
| 135 | } |
| 136 | if _, exists := got.Claim.Op.Journal["loss."+local.Details.ID]; exists { |
| 137 | t.Fatal("local-only declaration added recovery loss") |
| 138 | } |
| 139 | } |
| 140 | } |
| 141 | |
| 142 | func TestJ4DestroyedLocalOnlyRequiresExactAttestation(t *testing.T) { |
| 143 | for _, which := range []string{"missing", "wrong-retirement", "terminated-only", "stale-claim", "stale-record"} { |
| 144 | t.Run(which, func(t *testing.T) { |
| 145 | w := newLossWorld(t, "landing", func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { |
| 146 | c.Op.Phase = hostclaim.PhaseCommitted |
| 147 | r.Move.State = provision.MoveCommitted |
| 148 | c.Obligations = []hostclaim.Obligation{{ID: "local", Op: lossOp, Kind: "park-local", Box: "ie-box", Owner: "box:ie-box", Retirement: "ret"}} |
| 149 | }) |
| 150 | in := w.inspect(t, "ie-label") |
| 151 | sub := w.submission(in) |
| 152 | sub.Observation.Region = "eu-west-1" |
| 153 | sub.Observation.InstanceID = "i-ie" |
| 154 | switch which { |
| 155 | case "missing": |
| 156 | sub.Attestation.Disks = nil |
| 157 | case "wrong-retirement": |
| 158 | sub.Attestation.Disks = []string{lossOp + "/local/other"} |
| 159 | case "terminated-only": |
| 160 | sub.Observation.State = "terminated" |
| 161 | sub.Attestation.Disks = nil |
| 162 | case "stale-claim": |
| 163 | o := w.claim(t) |
| 164 | o.Claim.Obligations[0].Retirement = "other" |
| 165 | b, _ := json.Marshal(o.Claim) |
| 166 | if _, err := w.raw.Store.Put(context.Background(), w.claimK, b, o.ETag); err != nil { |
| 167 | t.Fatal(err) |
| 168 | } |
| 169 | case "stale-record": |
| 170 | r, tag := w.record(t) |
| 171 | r.RecordSeq++ |
| 172 | b, _ := json.Marshal(r) |
| 173 | if _, err := w.raw.Store.Put(context.Background(), w.key, b, tag); err != nil { |
| 174 | t.Fatal(err) |
| 175 | } |
| 176 | } |
| 177 | snap := lossSnapshot(t, w) |
| 178 | if _, err := w.control.DeclareMoveLoss(context.Background(), w.signer, lossBucket, "operator", "bab", "app", lossHost, "ie-box", sub); err == nil { |
| 179 | t.Fatal("unattested or stale discharge accepted") |
| 180 | } |
| 181 | if lossSnapshot(t, w) != snap { |
| 182 | t.Fatal("refusal wrote state") |
| 183 | } |
| 184 | }) |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | func TestJ4DestroyedLocalOnlyRejectsWidenedEffects(t *testing.T) { |
| 189 | w := newLossWorld(t, "landing", func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { |
| 190 | c.Op.Phase = hostclaim.PhaseCommitted |
| 191 | r.Move.State = provision.MoveCommitted |
| 192 | c.Obligations = []hostclaim.Obligation{{ID: "local", Op: "discovery", Kind: "park-local", Box: "ie-box", Retirement: "ret-local", Owner: "box:ie-box"}} |
| 193 | }) |
| 194 | c := w.claim(t).Claim |
| 195 | _, base, err := moveLocalLossConsequences(c, "ie-box") |
| 196 | if err != nil { |
| 197 | t.Fatal(err) |
| 198 | } |
| 199 | owner := &moveLossOwner{} |
| 200 | for name, change := range map[string]func(*hostclaim.LossConsequences){ |
| 201 | "execution": func(e *hostclaim.LossConsequences) { e.RevokeExecution = true }, |
| 202 | "participation": func(e *hostclaim.LossConsequences) { e.RemoveParticipation = true }, |
| 203 | "writes": func(e *hostclaim.LossConsequences) { e.WritesPotentiallyLost = true }, |
| 204 | "timers": func(e *hostclaim.LossConsequences) { e.TimersPotentiallyLost = true }, |
| 205 | "regional": func(e *hostclaim.LossConsequences) { e.ReassignRegional = e.DischargeLocal }, |
| 206 | "original-op": func(e *hostclaim.LossConsequences) { e.DischargeLocal[0].Op = "other" }, |
| 207 | "retirement": func(e *hostclaim.LossConsequences) { e.DischargeLocal[0].Retirement = "other" }, |
| 208 | "no-attestation": func(e *hostclaim.LossConsequences) { e.DestroyedStorage = nil }, |
| 209 | } { |
| 210 | t.Run(name, func(t *testing.T) { |
| 211 | b, _ := json.Marshal(base) |
| 212 | var effects hostclaim.LossConsequences |
| 213 | json.Unmarshal(b, &effects) |
| 214 | change(&effects) |
| 215 | if err := owner.ValidateDeclareLost(*c, hostclaim.LostDeclaration{Box: "ie-box", Consequences: effects}); err == nil { |
| 216 | t.Fatal("widened local-only consequences accepted") |
| 217 | } |
| 218 | }) |
| 219 | } |
| 220 | } |
| 221 | |