| 1 | package hostresolve |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "crypto/rand" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "github.com/carlosframework/platform/internal/hostactors" |
| 11 | "github.com/carlosframework/platform/internal/hostclaim" |
| 12 | "github.com/carlosframework/platform/internal/hostfence" |
| 13 | "github.com/carlosframework/platform/internal/hostfloor" |
| 14 | "github.com/carlosframework/platform/internal/hostquarantine" |
| 15 | "github.com/carlosframework/platform/internal/release" |
| 16 | "github.com/carlosframework/platform/internal/status" |
| 17 | "strings" |
| 18 | "testing" |
| 19 | "time" |
| 20 | ) |
| 21 | |
| 22 | type failClaimStore struct { |
| 23 | release.Store |
| 24 | fail bool |
| 25 | crash string |
| 26 | } |
| 27 | |
| 28 | func (s *failClaimStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) { |
| 29 | if s.fail && strings.HasPrefix(key, "control/hosts/") { |
| 30 | return "", errors.New("claim write unavailable") |
| 31 | } |
| 32 | if s.crash == "acceptance" && strings.HasPrefix(key, "control/host-loss-acceptances/") || s.crash == "before-quarantine" && key == hostactors.Key || s.crash == "loss-create" && strings.HasPrefix(key, "control/host-loss/") { |
| 33 | return "", errors.New("crash before durable write") |
| 34 | } |
| 35 | next, err := s.Store.Put(ctx, key, raw, etag) |
| 36 | if err == nil && s.crash == "after-quarantine" && key == hostactors.Key { |
| 37 | return "", errors.New("crash after durable quarantine") |
| 38 | } |
| 39 | return next, err |
| 40 | } |
| 41 | func resolveFixture(t *testing.T) (Service, *failClaimStore, *time.Time, string) { |
| 42 | t.Helper() |
| 43 | ctx := context.Background() |
| 44 | pub, key, _ := ed25519.GenerateKey(rand.Reader) |
| 45 | store := &failClaimStore{Store: release.NewDirStore(t.TempDir())} |
| 46 | signer := release.RootSigner(key) |
| 47 | now := time.Now().UTC() |
| 48 | d := hostactors.Directory{Store: store, PublicKey: pub, Bucket: "bucket"} |
| 49 | if err := d.Initialize(ctx, signer); err != nil { |
| 50 | t.Fatal(err) |
| 51 | } |
| 52 | o, err := d.Read(ctx) |
| 53 | if err != nil { |
| 54 | t.Fatal(err) |
| 55 | } |
| 56 | r := hostactors.Clone(o.Roster) |
| 57 | r.Boxes["dead"] = hostactors.Box{Enrollment: strings.Repeat("a", 32), Namespace: "platform", BoxID: "dead", Label: "dead-label", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-dead"}, EvidenceSHA256: strings.Repeat("b", 64), State: "active"} |
| 58 | if _, err = d.Replace(ctx, signer, o, r); err != nil { |
| 59 | t.Fatal(err) |
| 60 | } |
| 61 | host := "shop.bab.oncarlos.com" |
| 62 | observed, _ := hostclaim.Read(ctx, store, host) |
| 63 | n := 0 |
| 64 | _, err = hostclaim.Update(ctx, store, observed, hostclaim.Adopt(hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + host + ".json", Account: "bab", App: "jam", Host: host, Region: "ie", RecordRegion: "ie"}, "live", nil, []string{"dead"}, "discovery", func() string { n++; return fmt.Sprintf("id-%d", n) })) |
| 65 | if err != nil { |
| 66 | t.Fatal(err) |
| 67 | } |
| 68 | return Service{Directory: d, Signer: signer, Now: func() time.Time { return now }}, store, &now, host |
| 69 | } |
| 70 | func submitted(t *testing.T, s Service, now *time.Time, host string) hostfence.Submission { |
| 71 | t.Helper() |
| 72 | in, err := s.Inspect(context.Background(), "operator", host, "dead-label") |
| 73 | if err != nil { |
| 74 | t.Fatal(err) |
| 75 | } |
| 76 | *now = now.Add(2 * time.Second) |
| 77 | return hostfence.Submission{Challenge: in.Challenge, Confirmation: in.Details.ID, Observation: hostfence.CloudObservation{ObservedAt: in.Details.IssuedAt.Add(time.Second), Account: "123", Region: "eu-west-1", InstanceID: "i-dead", State: "stopped", RequestID: "request"}, Attestation: hostfence.OperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true, Disks: in.Details.Consequences.DestroyedStorageAssertion}} |
| 78 | } |
| 79 | func TestLostWiringPreservesServingAuthorityAndRetryEvidence(t *testing.T) { |
| 80 | ctx := context.Background() |
| 81 | s, store, now, host := resolveFixture(t) |
| 82 | in := submitted(t, s, now, host) |
| 83 | before, _ := hostclaim.Read(ctx, store, host) |
| 84 | after, err := s.Declare(ctx, "operator", host, in) |
| 85 | if err != nil { |
| 86 | t.Fatal(err) |
| 87 | } |
| 88 | if after.Claim.Op.Exec != before.Claim.Op.Exec || after.Claim.Holder.Stamp != before.Claim.Holder.Stamp || len(after.Claim.Obligations) != 0 { |
| 89 | t.Fatal(after.Claim) |
| 90 | } |
| 91 | loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) |
| 92 | if err != nil || loss.Record.State != "committed" { |
| 93 | t.Fatal(loss, err) |
| 94 | } |
| 95 | first := loss.Record.FencedAt |
| 96 | *now = now.Add(time.Hour) |
| 97 | if _, err = s.Declare(ctx, "operator", host, in); err != nil { |
| 98 | t.Fatal("accepted retry expired", err) |
| 99 | } |
| 100 | loss, _ = hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) |
| 101 | if !loss.Record.FencedAt.Equal(first) { |
| 102 | t.Fatal("retry moved fence time") |
| 103 | } |
| 104 | if _, err = s.Declare(ctx, "other-person", host, in); err == nil { |
| 105 | t.Fatal("actor changed") |
| 106 | } |
| 107 | } |
| 108 | func TestFailedClaimWriteLeavesGlobalQuarantine(t *testing.T) { |
| 109 | s, store, now, host := resolveFixture(t) |
| 110 | in := submitted(t, s, now, host) |
| 111 | store.fail = true |
| 112 | if _, err := s.Declare(context.Background(), "operator", host, in); err == nil { |
| 113 | t.Fatal("expected write error") |
| 114 | } |
| 115 | actor, err := s.Directory.Lookup(context.Background(), "dead") |
| 116 | if err != nil || actor.State != "quarantined" { |
| 117 | t.Fatal(actor, err) |
| 118 | } |
| 119 | store.fail = false |
| 120 | if _, err := s.Declare(context.Background(), "operator", host, in); err != nil { |
| 121 | t.Fatal(err) |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | func TestDeclareCrashAndChallengeExpiryAllowRetryAndReadmission(t *testing.T) { |
| 126 | for _, crash := range []string{"acceptance", "before-quarantine", "after-quarantine", "loss-create"} { |
| 127 | t.Run(crash, func(t *testing.T) { |
| 128 | ctx := t.Context() |
| 129 | s, store, now, host := resolveFixture(t) |
| 130 | in := submitted(t, s, now, host) |
| 131 | acceptedAt := *now |
| 132 | before, err := hostclaim.Read(ctx, store, host) |
| 133 | if err != nil { |
| 134 | t.Fatal(err) |
| 135 | } |
| 136 | store.crash = crash |
| 137 | if _, err := s.Declare(ctx, "operator", host, in); err == nil { |
| 138 | t.Fatal("crash not reached") |
| 139 | } |
| 140 | unchanged, err := hostclaim.Read(ctx, store, host) |
| 141 | if err != nil || unchanged.ETag != before.ETag { |
| 142 | t.Fatal("claim changed before quarantine completion", err) |
| 143 | } |
| 144 | roster, err := s.Directory.Read(ctx) |
| 145 | if err != nil { |
| 146 | t.Fatal(err) |
| 147 | } |
| 148 | b := roster.Roster.Boxes["dead"] |
| 149 | if crash == "acceptance" || crash == "before-quarantine" { |
| 150 | if b.State != "active" || len(b.LossIDs) != 0 { |
| 151 | t.Fatal("quarantined before acceptance", b) |
| 152 | } |
| 153 | } else if b.State != "quarantined" || len(b.LossIDs) != 1 || b.LossIDs[0] != in.Confirmation { |
| 154 | t.Fatal("deny not retained", b) |
| 155 | } |
| 156 | if _, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation); !errors.Is(err, hostclaim.ErrLossRecordAbsent) { |
| 157 | t.Fatal("loss unexpectedly materialized", err) |
| 158 | } |
| 159 | if crash != "acceptance" { |
| 160 | if _, err := s.readAcceptance(ctx, in.Confirmation); err != nil { |
| 161 | t.Fatal("acceptance missing", err) |
| 162 | } |
| 163 | } |
| 164 | store.crash = "" |
| 165 | if crash == "before-quarantine" { |
| 166 | // An unrelated roster CAS must not strand accepted evidence. |
| 167 | if _, err := s.Directory.Replace(ctx, s.Signer, roster, hostactors.Clone(roster.Roster)); err != nil { |
| 168 | t.Fatal(err) |
| 169 | } |
| 170 | } |
| 171 | *now = now.Add(time.Hour) |
| 172 | if _, err := hostfence.VerifyChallenge(s.Directory.PublicKey, in.Challenge, *now); err == nil { |
| 173 | t.Fatal("challenge did not expire") |
| 174 | } |
| 175 | if crash == "acceptance" { |
| 176 | if _, err := s.Declare(ctx, "operator", host, in); err == nil { |
| 177 | t.Fatal("unaccepted expired submission allowed") |
| 178 | } |
| 179 | return |
| 180 | } |
| 181 | if _, err := s.Declare(ctx, "operator", host, in); err != nil { |
| 182 | t.Fatal("accepted retry expired", err) |
| 183 | } |
| 184 | loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) |
| 185 | if err != nil || loss.Record.State != "committed" || !loss.Record.AcceptedAt.Equal(acceptedAt) || !loss.Record.FencedAt.Equal(acceptedAt) { |
| 186 | t.Fatal("acceptance changed", loss, err) |
| 187 | } |
| 188 | prepareReadmission(t, s, *now, loss.Record) |
| 189 | roster, err = s.Directory.Read(ctx) |
| 190 | if err != nil { |
| 191 | t.Fatal(err) |
| 192 | } |
| 193 | b = roster.Roster.Boxes["dead"] |
| 194 | if err := s.Readmit(ctx, "dead", b.Enrollment, b.QuarantineRevision, b.LossIDs); err != nil { |
| 195 | t.Fatal("orphan blocked readmission", err) |
| 196 | } |
| 197 | actor, err := s.Directory.Lookup(ctx, "dead") |
| 198 | if err != nil || actor.State != "active" { |
| 199 | t.Fatal("readmission failed", actor, err) |
| 200 | } |
| 201 | }) |
| 202 | } |
| 203 | } |
| 204 | |
| 205 | func prepareReadmission(t *testing.T, s Service, now time.Time, loss hostclaim.LossRecord) { |
| 206 | t.Helper() |
| 207 | ctx := t.Context() |
| 208 | store := s.Directory.Store |
| 209 | floor := hostfloor.Floor{Min: 1, MinRelease: "v1.2.3"} |
| 210 | if err := hostfloor.Publish(ctx, store, s.Signer, s.Directory.Bucket, floor); err != nil { |
| 211 | t.Fatal(err) |
| 212 | } |
| 213 | f, err := hostfloor.Read(ctx, store, s.Directory.PublicKey, s.Directory.Bucket) |
| 214 | if err != nil { |
| 215 | t.Fatal(err) |
| 216 | } |
| 217 | hash := strings.Repeat("d", 64) |
| 218 | rel := release.SystemRelease{Version: "build123", ReleaseTag: floor.MinRelease, MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{"linux-amd64": {Key: release.SystemArtifactKey("build123", "linux-amd64"), SHA256: hash}}} |
| 219 | if _, err := release.PromoteSystem(ctx, store, s.Signer, "router", "stable", rel, release.SystemPromoteOptions{}); err != nil { |
| 220 | t.Fatal(err) |
| 221 | } |
| 222 | wire, _, err := store.Get(ctx, release.SystemPointerKey("router", "stable")) |
| 223 | if err != nil { |
| 224 | t.Fatal(err) |
| 225 | } |
| 226 | if err := release.PublishSystemIdentity(ctx, store, s.Directory.PublicKey, wire); err != nil { |
| 227 | t.Fatal(err) |
| 228 | } |
| 229 | receipt := hostquarantine.RetirementCompletion{V: 1, Box: loss.Box, Enrollment: loss.Enrollment, InstanceID: loss.CloudObservation.InstanceID, QuarantineRevision: loss.QuarantineRevision, BootID: "new-boot", CompletedAt: now, Complete: true, InventorySHA256: strings.Repeat("e", 64), Entries: []hostquarantine.RetirementEntry{{Host: loss.Host, Incarnation: loss.HolderIncarnation, Stamp: loss.HolderStamp, RetirementID: "retirement", JournalSHA256: strings.Repeat("f", 64)}}} |
| 230 | if err := receipt.Validate(); err != nil { |
| 231 | t.Fatal(err) |
| 232 | } |
| 233 | raw, err := json.Marshal(receipt) |
| 234 | if err != nil { |
| 235 | t.Fatal(err) |
| 236 | } |
| 237 | key, err := hostquarantine.RetirementKey(loss.Box, loss.Enrollment, loss.QuarantineRevision) |
| 238 | if err != nil { |
| 239 | t.Fatal(err) |
| 240 | } |
| 241 | if err := release.CreateExact(ctx, store, key, raw); err != nil { |
| 242 | t.Fatal(err) |
| 243 | } |
| 244 | report := status.Box{GeneratedAt: now, Label: "dead-label", Metrics: &status.BoxMetrics{Enrollment: loss.Enrollment, InstanceID: loss.CloudObservation.InstanceID, Release: rel.Version, ReleaseTag: rel.ReleaseTag, Platform: "linux-amd64", BinarySHA256: hash, MovesProtocol: 1, BootID: receipt.BootID, FloorSHA256: f.Digest, FloorGuardReady: true, HostClaimEnabled: true, RetirementReceiptSHA256: hostfloor.Digest(raw)}} |
| 245 | raw, err = json.Marshal(report) |
| 246 | if err != nil { |
| 247 | t.Fatal(err) |
| 248 | } |
| 249 | if err := release.CreateExact(ctx, store, "status/boxes/"+loss.Box+".json", raw); err != nil { |
| 250 | t.Fatal(err) |
| 251 | } |
| 252 | } |
| 253 | |
amadan