| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "net/http/httptest" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/hostbox" |
| 14 | "github.com/carlosframework/platform/internal/hostclaim" |
| 15 | execlease "github.com/carlosframework/platform/internal/hostexec" |
| 16 | "github.com/carlosframework/platform/internal/hostguard" |
| 17 | "github.com/carlosframework/platform/internal/hostops" |
| 18 | "github.com/carlosframework/platform/internal/lease/leasetest" |
| 19 | "github.com/carlosframework/platform/internal/park" |
| 20 | ) |
| 21 | |
| 22 | func TestPlan14OrdinaryGenerationParks(t *testing.T) { |
| 23 | for _, tc := range []string{"valid", "real-leases", "moves", "foreign-region", "foreign-box", "current-generation", "unknown-generation", "missing-receipt", "altered-receipt", "foreign-enrollment", "unbound-root", "changed-time", "changed-location", "unknown-obligation"} { |
| 24 | t.Run(tc, func(t *testing.T) { |
| 25 | f, _, _, in := servingRetirementFixture(t) |
| 26 | first := servingAdmit(t, f, &in, "G") |
| 27 | receipt := servingStop(t, f, first) |
| 28 | current := servingAdmit(t, f, &in, "G2") |
| 29 | current, err := hostclaim.Update(t.Context(), f.claims, current, hostclaim.PublishGeneration("", "", current.Claim.Holder.Session, "G2", "ie", f.newID)) |
| 30 | if err != nil { |
| 31 | t.Fatal(err) |
| 32 | } |
| 33 | var ob hostclaim.Obligation |
| 34 | for _, x := range current.Claim.Obligations { |
| 35 | if x.Generation == "G" { |
| 36 | ob = x |
| 37 | } |
| 38 | } |
| 39 | if ob.ID == "" { |
| 40 | t.Fatal("missing publication obligation") |
| 41 | } |
| 42 | identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "box-1", Enrollment: "enrollment", Revision: 7, Binding: "7", Region: "ie", FleetHomeRegion: "ie"}) |
| 43 | identity.HomeRegion = "ie" |
| 44 | if tc == "foreign-enrollment" { |
| 45 | receipt.Target.Enrollment = "another-enrollment" |
| 46 | var proof StopReceipt |
| 47 | if err := json.Unmarshal(receipt.Bytes, &proof); err != nil { |
| 48 | t.Fatal(err) |
| 49 | } |
| 50 | proof.Enrollment = receipt.Target.Enrollment |
| 51 | receipt.Bytes = mustMarshalStopReceipt(proof) |
| 52 | } |
| 53 | guard := f.guard |
| 54 | guard.FenceView = func(_ context.Context, host string, c hostguard.Captured) (hostguard.FenceView, error) { |
| 55 | receipts := []hostguard.FenceReceipt{receipt} |
| 56 | if tc == "missing-receipt" { |
| 57 | receipts = nil |
| 58 | } |
| 59 | if tc == "altered-receipt" { |
| 60 | receipts[0].Bytes = []byte("forged") |
| 61 | } |
| 62 | return hostguard.FenceView{Schema: 1, Host: host, Captured: c, Installation: hostguard.RootInstallation{Host: host, Migration: 1}, Receipts: receipts}, nil |
| 63 | } |
| 64 | data := leasetest.NewVersionedBucket() |
| 65 | data.AddBucket("data") |
| 66 | p := productionClaimRegional{Identity: identity, Local: &productionClaimLocal{assembly: &ClaimAssembly{Guard: guard, Data: data}}} |
| 67 | switch tc { |
| 68 | case "moves": |
| 69 | current.Claim.Op.Kind = hostclaim.KindMoving |
| 70 | case "foreign-region": |
| 71 | ob.Region, ob.Owner = "uk", "region:uk" |
| 72 | case "foreign-box": |
| 73 | current.Claim.Holder.Admissions[0].Box = "other" |
| 74 | case "current-generation": |
| 75 | ob.Generation = "G2" |
| 76 | case "unknown-generation": |
| 77 | ob.Generation = "unknown" |
| 78 | case "unknown-obligation": |
| 79 | ob.ID = "unknown" |
| 80 | case "unbound-root": |
| 81 | p.Local.assembly.Guard.FenceView = nil |
| 82 | case "changed-time", "changed-location": |
| 83 | scope, err := p.VerifyObligation(t.Context(), current, ob) |
| 84 | if err != nil { |
| 85 | t.Fatal(err) |
| 86 | } |
| 87 | if tc == "changed-time" { |
| 88 | scope.EvidenceAt = scope.EvidenceAt.Add(1) |
| 89 | } else { |
| 90 | scope.Location += "/other" |
| 91 | } |
| 92 | ob.Park = &scope |
| 93 | |
| 94 | } |
| 95 | |
| 96 | if tc == "foreign-region" || tc == "current-generation" || tc == "unknown-generation" { |
| 97 | for i := range current.Claim.Obligations { |
| 98 | if current.Claim.Obligations[i].ID == ob.ID { |
| 99 | current.Claim.Obligations[i] = ob |
| 100 | } |
| 101 | } |
| 102 | } |
| 103 | scope, err := p.VerifyObligation(t.Context(), current, ob) |
| 104 | if tc == "valid" || tc == "real-leases" { |
| 105 | if err != nil || scope.Location != current.Claim.Holder.Location || !scope.EvidenceAt.Equal(receipt.EvidenceAt) || scope.RequestLimit != hostbox.RequestLimit { |
| 106 | t.Fatal(scope, err) |
| 107 | } |
| 108 | ob.Park = &scope |
| 109 | p.Local.Identity, p.Local.lifetime = identity, t.Context() |
| 110 | p.Local.Resources.Mode = "agent" |
| 111 | p.Local.executionCall = func(context.Context, execlease.ExecutionRequest) (execlease.ExecutionResponse, error) { |
| 112 | return execlease.ExecutionResponse{}, errors.New("stopped current writer cannot execute") |
| 113 | } |
| 114 | if err := p.Authorize(t.Context(), current, ob); err != nil { |
| 115 | t.Fatal("cleanup required writer execution permission", err) |
| 116 | } |
| 117 | if _, err = p.VerifyObligation(t.Context(), current, ob); err != nil { |
| 118 | t.Fatal("bound replay", err) |
| 119 | } |
| 120 | if _, err = p.VerifyMovesSweep(t.Context(), current, nil); err == nil { |
| 121 | t.Fatal("moves sweep enabled") |
| 122 | } |
| 123 | freed := current |
| 124 | claim := *current.Claim |
| 125 | freed.Claim = &claim |
| 126 | claim.RetiredIncarnations = []string{claim.Holder.Incarnation} |
| 127 | claim.Holder, claim.Op = nil, hostclaim.Op{} |
| 128 | claim.QuietAfter = map[string]time.Time{"ie": receipt.EvidenceAt} |
| 129 | if _, err = p.VerifyMovesSweep(t.Context(), freed, nil); err == nil { |
| 130 | t.Fatal("free claim enabled moves sweep") |
| 131 | } |
| 132 | location := current.Claim.Holder.Location |
| 133 | oldSnapshot := location + "/g-G/0009/snapshot.ltx" |
| 134 | oldVersion := data.Put("data", oldSnapshot, []byte("old snapshot")) |
| 135 | data.Put("data", location+"/g-G2/0009/snapshot.ltx", []byte("current snapshot")) |
| 136 | keptFile := location + "/files/g-G/objects/0000000000000001-aa" |
| 137 | keptVersion := data.Put("data", keptFile, []byte("kept")) |
| 138 | filesManifest(t, func(k string, b []byte) { data.Put("data", k, b) }, location, "G2", "kept.txt="+keptFile) |
| 139 | marker, err := json.Marshal(map[string]any{"host": current.Host, "incarnation": current.Claim.Holder.Incarnation, "op": "G2"}) |
| 140 | if err != nil { |
| 141 | t.Fatal(err) |
| 142 | } |
| 143 | data.Put("data", location+"/generation.json", marker) |
| 144 | engine := park.Engine{HomeRegion: "ie", Store: data, Claims: f.claims, Bucket: "data", Region: "ie", Leases: evLeases{}, Now: func() time.Time { return f.now.Add(park.SignatureAge + hostbox.RequestLimit + time.Second) }, NewID: f.newID, View: p.View, Authorize: p.Authorize} |
| 145 | if tc == "real-leases" { |
| 146 | handler, _ := leasetest.NewS3Server(data) |
| 147 | server := httptest.NewServer(handler) |
| 148 | defer server.Close() |
| 149 | for key, value := range map[string]string{"CARLOS_S3_ENDPOINT": server.URL, "AWS_ENDPOINT_URL_S3": server.URL, "AWS_ACCESS_KEY_ID": "test-key", "AWS_SECRET_ACCESS_KEY": "test-secret", "AWS_SESSION_TOKEN": "", "AWS_EC2_METADATA_DISABLED": "true", "AWS_CONFIG_FILE": "/nonexistent", "AWS_SHARED_CREDENTIALS_FILE": "/nonexistent"} { |
| 150 | t.Setenv(key, value) |
| 151 | } |
| 152 | engine.Leases = hostbox.AggregateLeases{Leases: hostbox.RenewingLeases{Region: "eu-west-1"}, Resolve: p.LeaseSet} |
| 153 | } |
| 154 | work := park.Work{Host: current.Host, ID: ob.ID, Op: ob.Op} |
| 155 | if err := engine.Bind(t.Context(), work, scope); err != nil { |
| 156 | t.Fatal("bind generation", err) |
| 157 | } |
| 158 | pinnedMarker, err := json.Marshal(map[string]any{"host": current.Host, "incarnation": current.Claim.Holder.Incarnation, "op": "G2", "snapshot_key": oldSnapshot, "snapshot_version": oldVersion}) |
| 159 | if err != nil { |
| 160 | t.Fatal(err) |
| 161 | } |
| 162 | data.Put("data", location+"/generation.json", pinnedMarker) |
| 163 | if err := engine.Run(t.Context(), work); !errors.Is(err, park.ErrUnsafe) { |
| 164 | t.Fatal("database pin did not protect its generation", err) |
| 165 | } |
| 166 | if _, err := data.GetVersion(t.Context(), "data", oldSnapshot, oldVersion); err != nil { |
| 167 | t.Fatal("pinned database snapshot was deleted", err) |
| 168 | } |
| 169 | data.Put("data", location+"/generation.json", marker) |
| 170 | if err := engine.Run(t.Context(), work); err != nil { |
| 171 | t.Fatal("retire generation", err) |
| 172 | } |
| 173 | left, err := data.ListVersions(t.Context(), "data", location+"/g-G/") |
| 174 | if err != nil || len(left) != 0 { |
| 175 | t.Fatal("predecessor remains", left, err) |
| 176 | } |
| 177 | left, err = data.ListVersions(t.Context(), "data", location+"/g-G2/") |
| 178 | if err != nil || len(left) != 1 { |
| 179 | t.Fatal("current generation changed", left, err) |
| 180 | } |
| 181 | if body, err := data.GetVersion(t.Context(), "data", keptFile, keptVersion); err != nil || string(body) != "kept" { |
| 182 | t.Fatal("referenced file was retired with its database generation", err) |
| 183 | } |
| 184 | |
| 185 | } else if err == nil { |
| 186 | t.Fatal("accepted", tc) |
| 187 | } |
| 188 | }) |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | func TestPlan14OrdinaryRetirementRequiresProof(t *testing.T) { |
| 193 | t.Parallel() |
| 194 | for _, target := range []string{"holder", "empty-staging", "staging-residue", "foreign"} { |
| 195 | t.Run(target, func(t *testing.T) { |
| 196 | f, _, _, in := servingRetirementFixture(t) |
| 197 | o := servingAdmit(t, f, &in, "G") |
| 198 | identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "box-1", Enrollment: "enrollment", Revision: 7, Binding: "7", Region: "ie", FleetHomeRegion: "ie"}) |
| 199 | identity.HomeRegion = "ie" |
| 200 | data := leasetest.NewVersionedBucket() |
| 201 | data.AddBucket("data") |
| 202 | ob := hostclaim.Obligation{ID: "park", Op: "delete", Kind: "park-replica", Owner: "region:ie", Region: "ie", Retirement: "retire", ParkTarget: o.Claim.Holder.Location} |
| 203 | want := hostops.ErrIdentity |
| 204 | switch target { |
| 205 | case "empty-staging", "staging-residue": |
| 206 | ob.ParkTarget = "moves/" + o.Host |
| 207 | data.Put("data", ob.ParkTarget+"/lock.json", []byte("lock")) |
| 208 | if target == "staging-residue" { |
| 209 | data.Put("data", ob.ParkTarget+"/other/g-G/object", []byte("deferred move")) |
| 210 | want = park.ErrUnsafe |
| 211 | } |
| 212 | case "foreign": |
| 213 | ob.ParkTarget += "/another" |
| 214 | want = park.ErrUnsafe |
| 215 | } |
| 216 | o.Claim.Op = hostclaim.Op{Kind: hostclaim.KindParking, ID: "delete"} |
| 217 | o.Claim.Obligations = []hostclaim.Obligation{ob} |
| 218 | if err := o.Claim.Validate(); err != nil { |
| 219 | t.Fatal(err) |
| 220 | } |
| 221 | p := productionClaimRegional{Identity: identity, Local: &productionClaimLocal{assembly: &ClaimAssembly{Data: data, RetirementAuthority: &hostops.ProductionAuthority{}}}} |
| 222 | if err := p.authorizeOrdinaryRetirement(t.Context(), o, ob); !errors.Is(err, want) { |
| 223 | t.Fatal("unsigned retirement or deferred target accepted", err) |
| 224 | } |
| 225 | }) |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | func TestPlan14OrdinaryParkView(t *testing.T) { |
| 230 | t.Parallel() |
| 231 | f, _, _, in := servingRetirementFixture(t) |
| 232 | o := servingAdmit(t, f, &in, "G") |
| 233 | identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "box-1", Enrollment: "enrollment", Revision: 7, Binding: "7", Region: "ie", FleetHomeRegion: "ie"}) |
| 234 | identity.HomeRegion = "ie" |
| 235 | data := leasetest.NewVersionedBucket() |
| 236 | data.AddBucket("data") |
| 237 | p := productionClaimRegional{Identity: identity, Local: &productionClaimLocal{assembly: &ClaimAssembly{Data: data}}} |
| 238 | v, err := p.View(t.Context(), o) |
| 239 | if err != nil { |
| 240 | t.Fatal(err) |
| 241 | } |
| 242 | live, err := park.LiveGenerations(o.Claim, v, o.Claim.Holder.Location) |
| 243 | if err != nil || !live["G"] || !live[hostclaim.RootGeneration] || v.RefusedLocations["moves/"+o.Host] == nil { |
| 244 | t.Fatal(live, err) |
| 245 | } |
| 246 | |
| 247 | p.Identity.HomeRegion = "" |
| 248 | v, err = p.View(t.Context(), o) |
| 249 | if err != nil || v.HomeRegion != "" || v.AuthoritativeRegion != "ie" { |
| 250 | t.Fatal("named home placement", v, err) |
| 251 | } |
| 252 | p.Identity.HomeRegion = "ie" |
| 253 | key := o.Claim.Holder.Location + "/g-G/0009/snapshot.ltx" |
| 254 | version := data.Put("data", key, []byte("retained snapshot")) |
| 255 | marker, err := json.Marshal(map[string]any{"host": o.Host, "incarnation": o.Claim.Holder.Incarnation, "op": "G", "snapshot_key": key, "snapshot_version": version}) |
| 256 | if err != nil { |
| 257 | t.Fatal(err) |
| 258 | } |
| 259 | data.Put("data", o.Claim.Holder.Location+"/generation.json", marker) |
| 260 | data.Put("data", key, []byte("newer bytes")) |
| 261 | v, err = p.View(t.Context(), o) |
| 262 | if err != nil || !v.Pinned[park.Ref{Key: key, VersionID: version}] || v.MarkerGeneration != "G" { |
| 263 | t.Fatal("lost exact snapshot pin", v, err) |
| 264 | } |
| 265 | unavailable := errors.New("version read unavailable") |
| 266 | data.FailOn = func(op, key string) error { |
| 267 | if op == "get" { |
| 268 | return unavailable |
| 269 | } |
| 270 | return nil |
| 271 | } |
| 272 | if _, err := p.View(t.Context(), o); !errors.Is(err, unavailable) { |
| 273 | t.Fatal("reduced live set on failed read", err) |
| 274 | } |
| 275 | data.FailOn = nil |
| 276 | for _, kind := range []string{hostclaim.KindMoving, hostclaim.KindLanding, hostclaim.KindRelocating, hostclaim.KindRestoring} { |
| 277 | o.Claim.Op.Kind = kind |
| 278 | if _, err := p.View(t.Context(), o); err == nil { |
| 279 | t.Fatal("move view enabled", kind) |
| 280 | } |
| 281 | } |
| 282 | } |
| 283 | |
amadan