| 1 | package hostactors |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "github.com/carlosframework/platform/internal/hostclaim" |
| 8 | "github.com/carlosframework/platform/internal/hostfloor" |
| 9 | "github.com/carlosframework/platform/internal/release" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | "time" |
| 13 | ) |
| 14 | |
| 15 | // writeFence records ref's termination as observed by a separate regional |
| 16 | // writer box. The writer sits in another pool, so no census scope requires |
| 17 | // it; a box never witnesses its own termination (M1-5). |
| 18 | func writeFence(t *testing.T, d Directory, o *Observed, ref string, now time.Time, invalid bool) { |
| 19 | t.Helper() |
| 20 | b := o.Roster.Boxes[ref] |
| 21 | writer := b |
| 22 | writer.BoxID, writer.Label, writer.Pool = "fence-writer", "fence-writer", "workers" |
| 23 | writer.Enrollment = strings.Repeat("9", 32) |
| 24 | writer.Cloud.InstanceID = "i-fence-writer" |
| 25 | o.Roster.Boxes["fence-writer"] = writer |
| 26 | state := "terminated" |
| 27 | if invalid { |
| 28 | state = "stopped" |
| 29 | } |
| 30 | f := map[string]any{"v": 1, "box": ref, "enrollment": b.Enrollment, "cloud": b.Cloud, "state": state, "observed_at": now, "via": "describe", "request_id": "request", "written_by": map[string]string{"box": "fence-writer", "enrollment": writer.Enrollment}} |
| 31 | raw, _ := json.Marshal(f) |
| 32 | if err := release.CreateExact(context.Background(), d.Store, "control/box-fences/"+b.Enrollment+".json", raw); err != nil { |
| 33 | t.Fatal(err) |
| 34 | } |
| 35 | } |
| 36 | func TestTerminationFenceExcludesActiveBeforeRosterUpdate(t *testing.T) { |
| 37 | for _, invalid := range []bool{false, true} { |
| 38 | t.Run(map[bool]string{false: "valid", true: "invalid"}[invalid], func(t *testing.T) { |
| 39 | d, o, e, f, scope, now := censusFixture(t) |
| 40 | writeFence(t, d, &o, "box", now, invalid) |
| 41 | reports, err := Census(context.Background(), o, e, d.PublicKey, f, scope, now) |
| 42 | if invalid { |
| 43 | if err == nil { |
| 44 | t.Fatal("invalid fence treated as absence or exclusion") |
| 45 | } |
| 46 | } else if err != nil || len(reports) != 1 { |
| 47 | t.Fatal(reports, err) |
| 48 | } |
| 49 | if _, err := d.Lookup(context.Background(), "box"); err == nil { |
| 50 | t.Fatal("fenced or invalid actor eligible") |
| 51 | } |
| 52 | if _, err := d.CheckBoxStart(context.Background(), "box"); err == nil { |
| 53 | t.Fatal("fenced or invalid start allowed") |
| 54 | } |
| 55 | scope.Participants = []string{"box"} |
| 56 | if _, err := Census(context.Background(), o, e, d.PublicKey, f, scope, now); err == nil { |
| 57 | t.Fatal("fenced participant allowed") |
| 58 | } |
| 59 | }) |
| 60 | } |
| 61 | } |
| 62 | func boundConsole(o *Observed, e *reportsFixture) { |
| 63 | c := o.Roster.Consoles["console-1"] |
| 64 | c.Compute = &ComputeBinding{Box: "box", Enrollment: e.box.Enrollment} |
| 65 | o.Roster.Consoles["console-1"] = c |
| 66 | e.console.Compute = c.Compute |
| 67 | e.box.ConsoleProcesses = []ConsoleProcess{{ConsoleID: "console-1", Enrollment: c.Enrollment, ProcessID: e.console.ProcessID, BinarySHA256: e.console.BinarySHA256, GuardFloorSHA256: e.box.FloorSHA256, AllLaunchPathsGuarded: true}} |
| 68 | } |
| 69 | func TestConsoleFenceRequiresExactProcessMachineAndBoot(t *testing.T) { |
| 70 | for _, fault := range []string{"good", "missing", "boot", "enrollment", "process", "ambiguous", "console-enrollment", "console-instance"} { |
| 71 | t.Run(fault, func(t *testing.T) { |
| 72 | d, o, e, f, scope, now := censusFixture(t) |
| 73 | boundConsole(&o, &e) |
| 74 | // The starter lives elsewhere; only the bound console can be excluded. |
| 75 | scope.ConsoleID = "starter" |
| 76 | c := o.Roster.Consoles["console-1"] |
| 77 | c.Compute = nil |
| 78 | c.InstanceID = "starter" |
| 79 | c.Enrollment = "dddddddddddddddddddddddddddddddd" |
| 80 | o.Roster.Consoles["starter"] = c |
| 81 | switch fault { |
| 82 | case "missing": |
| 83 | c = o.Roster.Consoles["console-1"] |
| 84 | c.Compute = nil |
| 85 | o.Roster.Consoles["console-1"] = c |
| 86 | case "console-enrollment": |
| 87 | e.console.Enrollment = "wrong" |
| 88 | case "console-instance": |
| 89 | e.console.InstanceID = "wrong" |
| 90 | case "boot": |
| 91 | e.console.BootID = "other" |
| 92 | case "enrollment": |
| 93 | e.console.Compute = &ComputeBinding{Box: "box", Enrollment: "other"} |
| 94 | case "process": |
| 95 | e.box.ConsoleProcesses[0].ProcessID = "other" |
| 96 | case "ambiguous": |
| 97 | e.box.ConsoleProcesses = append(e.box.ConsoleProcesses, e.box.ConsoleProcesses[0]) |
| 98 | } |
| 99 | // Validate the binding independently, then exercise full census below with exact objects. |
| 100 | _, err := verifyCompute(context.Background(), o, e, "console-1", o.Roster.Consoles["console-1"], e.console) |
| 101 | if (err == nil) != (fault == "good") { |
| 102 | t.Fatal(fault, err) |
| 103 | } |
| 104 | if fault != "good" { |
| 105 | return |
| 106 | } |
| 107 | writeFence(t, d, &o, "box", now, false) |
| 108 | for id, cc := range o.Roster.Consoles { |
| 109 | r := e.console |
| 110 | if id == "starter" { |
| 111 | r.Actor = id |
| 112 | r.Enrollment = cc.Enrollment |
| 113 | r.InstanceID = id |
| 114 | r.Compute = nil |
| 115 | } |
| 116 | raw, _ := json.Marshal(r) |
| 117 | if err := release.CreateExact(context.Background(), d.Store, "status/consoles/"+id+".json", raw); err != nil { |
| 118 | t.Fatal(err) |
| 119 | } |
| 120 | } |
| 121 | local := e.console |
| 122 | local.Actor = "starter" |
| 123 | local.Enrollment = c.Enrollment |
| 124 | local.InstanceID = "starter" |
| 125 | local.Compute = nil |
| 126 | exact := ExactEvidence{Store: d.Store, PublicKey: d.PublicKey, Boxes: e.BoxReport, Self: func(context.Context) (Report, error) { return local, nil }} |
| 127 | reports, err := Census(context.Background(), o, exact, d.PublicKey, f, scope, now) |
| 128 | if err != nil || len(reports) != 1 || reports["console:starter"].Actor != "starter" { |
| 129 | t.Fatal(reports, err) |
| 130 | } |
| 131 | }) |
| 132 | } |
| 133 | } |
| 134 | func TestRelocationOnlyNewestGuardedConsoleMayBeStale(t *testing.T) { |
| 135 | for _, fault := range []string{"good", "ordinary", "guard", "floor", "binding", "future", "newer-malformed", "newer-image", "other-operation"} { |
| 136 | t.Run(fault, func(t *testing.T) { |
| 137 | d, o, e, f, scope, now := censusFixture(t) |
| 138 | boundConsole(&o, &e) |
| 139 | e.console.GeneratedAt = now.Add(-time.Hour) |
| 140 | scope.boxRelocation = true |
| 141 | scope.StarterBox = "box" |
| 142 | scope.ConsoleID = "" |
| 143 | switch fault { |
| 144 | case "ordinary", "other-operation": |
| 145 | scope.boxRelocation = false |
| 146 | scope.ConsoleID = "console-1" |
| 147 | case "guard": |
| 148 | e.box.ConsoleProcesses[0].AllLaunchPathsGuarded = false |
| 149 | case "floor": |
| 150 | e.box.ConsoleProcesses[0].GuardFloorSHA256 = "wrong" |
| 151 | case "binding": |
| 152 | e.console.Compute = nil |
| 153 | case "future": |
| 154 | e.console.GeneratedAt = now.Add(time.Minute) |
| 155 | case "newer-image": |
| 156 | e.console.BinarySHA256 = "wrong" |
| 157 | } |
| 158 | raw, _ := json.Marshal(e.console) |
| 159 | if fault == "newer-malformed" { |
| 160 | raw = []byte("{") |
| 161 | } |
| 162 | if err := release.CreateExact(context.Background(), d.Store, "status/consoles/console-1.json", raw); err != nil { |
| 163 | t.Fatal(err) |
| 164 | } |
| 165 | exact := ExactEvidence{Store: d.Store, PublicKey: d.PublicKey, Boxes: e.BoxReport, Self: func(context.Context) (Report, error) { return e.box, nil }} |
| 166 | _, err := Census(context.Background(), o, exact, d.PublicKey, f, scope, now) |
| 167 | if (err == nil) != (fault == "good") { |
| 168 | t.Fatal(fault, err) |
| 169 | } |
| 170 | }) |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | type brokenFenceStore struct{ release.Store } |
| 175 | |
| 176 | func (s brokenFenceStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 177 | if strings.HasPrefix(key, "control/box-fences/") { |
| 178 | return nil, "", errors.New("fence read unavailable") |
| 179 | } |
| 180 | return s.Store.Get(ctx, key) |
| 181 | } |
| 182 | func TestFenceAuthorityErrorsAndMismatchesRefuse(t *testing.T) { |
| 183 | for _, fault := range []string{"read", "malformed", "enrollment", "cloud", "writer", "future"} { |
| 184 | t.Run(fault, func(t *testing.T) { |
| 185 | d, o, e, f, scope, now := censusFixture(t) |
| 186 | if fault == "read" { |
| 187 | d.Store = brokenFenceStore{d.Store} |
| 188 | e.d = d |
| 189 | } else { |
| 190 | writeFence(t, d, &o, "box", now, false) |
| 191 | key := "control/box-fences/" + o.Roster.Boxes["box"].Enrollment + ".json" |
| 192 | raw, etag, err := d.Store.Get(context.Background(), key) |
| 193 | if err != nil { |
| 194 | t.Fatal(err) |
| 195 | } |
| 196 | var record map[string]any |
| 197 | if err := json.Unmarshal(raw, &record); err != nil { |
| 198 | t.Fatal(err) |
| 199 | } |
| 200 | switch fault { |
| 201 | case "enrollment": |
| 202 | record["enrollment"] = "wrong" |
| 203 | case "cloud": |
| 204 | record["cloud"] = Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "wrong"} |
| 205 | case "writer": |
| 206 | record["written_by"] = map[string]string{"box": "unknown", "enrollment": "wrong"} |
| 207 | case "future": |
| 208 | record["observed_at"] = now.Add(time.Minute) |
| 209 | } |
| 210 | raw, _ = json.Marshal(record) |
| 211 | if fault == "malformed" { |
| 212 | raw = []byte("{") |
| 213 | } |
| 214 | // Test-only mutation creates corrupt authority without allowing production overwrite. |
| 215 | if _, err = d.Store.Put(context.Background(), key, raw, etag); err != nil { |
| 216 | t.Fatal(err) |
| 217 | } |
| 218 | } |
| 219 | if _, err := Census(context.Background(), o, e, d.PublicKey, f, scope, now); err == nil { |
| 220 | t.Fatal("bad authority admitted census") |
| 221 | } |
| 222 | if _, err := d.Lookup(context.Background(), "box"); err == nil { |
| 223 | t.Fatal("bad authority granted eligibility") |
| 224 | } |
| 225 | if _, err := d.CheckBoxStart(context.Background(), "box"); err == nil { |
| 226 | t.Fatal("bad authority granted start") |
| 227 | } |
| 228 | }) |
| 229 | } |
| 230 | } |
| 231 | |
| 232 | func TestStartRevalidatesReservedActorEligibility(t *testing.T) { |
| 233 | for _, fault := range []string{"good", "quarantined", "retired", "fenced", "malformed", "unreadable", "console-quarantined", "console-fenced"} { |
| 234 | t.Run(fault, func(t *testing.T) { |
| 235 | ctx := t.Context() |
| 236 | d, sg, o := fixture(t) |
| 237 | floor := hostfloor.Floor{Min: 1, MinRelease: "v1.2.3"} |
| 238 | if err := hostfloor.Publish(ctx, d.Store, sg, d.Bucket, floor); err != nil { |
| 239 | t.Fatal(err) |
| 240 | } |
| 241 | f, err := hostfloor.Read(ctx, d.Store, d.PublicKey, d.Bucket) |
| 242 | if err != nil { |
| 243 | t.Fatal(err) |
| 244 | } |
| 245 | r := Clone(o.Roster) |
| 246 | r.Publication, r.FloorSHA256 = "published", f.Digest |
| 247 | actor, enrollment := "box:box", r.Boxes["box"].Enrollment |
| 248 | if strings.HasPrefix(fault, "console-") { |
| 249 | actor, enrollment = "console:console-1", strings.Repeat("c", 32) |
| 250 | r.Consoles["console-1"] = Console{Enrollment: enrollment, InstanceID: "console-1", Namespaces: []string{"platform"}, DeploymentBucket: d.Bucket, Compute: &ComputeBinding{Box: "box", Enrollment: r.Boxes["box"].Enrollment}} |
| 251 | } |
| 252 | o, err = d.Replace(ctx, sg, o, r) |
| 253 | if err != nil { |
| 254 | t.Fatal(err) |
| 255 | } |
| 256 | before, err := hostclaim.Read(ctx, d.Store, "shop.example.com") |
| 257 | if err != nil { |
| 258 | t.Fatal(err) |
| 259 | } |
| 260 | h := Reservation{ID: "hold", Host: before.Host, OpID: "operation", ExpectedClaimETag: before.ETag, FloorSHA256: f.Digest, Actors: map[string]string{actor: enrollment}, Releases: map[string]Report{actor: {Enrollment: enrollment, GeneratedAt: time.Now()}}} |
| 261 | o, err = d.Reserve(ctx, sg, o, h, false) |
| 262 | if err != nil { |
| 263 | t.Fatal(err) |
| 264 | } |
| 265 | switch fault { |
| 266 | case "quarantined", "console-quarantined": |
| 267 | _, err = d.Quarantine(ctx, sg, o, "box", o.Roster.Boxes["box"].Enrollment, "loss") |
| 268 | case "retired": |
| 269 | r = Clone(o.Roster) |
| 270 | b := r.Boxes["box"] |
| 271 | b.State = "retired" |
| 272 | r.Boxes["box"] = b |
| 273 | _, err = d.Replace(ctx, sg, o, r) |
| 274 | case "fenced", "console-fenced", "malformed": |
| 275 | writeFence(t, d, &o, "box", time.Now(), fault == "malformed") |
| 276 | case "unreadable": |
| 277 | d.Store = brokenFenceStore{d.Store} |
| 278 | } |
| 279 | if err != nil { |
| 280 | t.Fatal(err) |
| 281 | } |
| 282 | called := false |
| 283 | _, err = d.Start(ctx, before, h.ID, func(*hostclaim.Claim) error { called = true; return errors.New("stop before writing") }) |
| 284 | if err == nil || called != (fault == "good") { |
| 285 | t.Fatalf("transition called=%v, error=%v", called, err) |
| 286 | } |
| 287 | after, err := hostclaim.Read(ctx, d.Store, before.Host) |
| 288 | if err != nil || before.ETag != after.ETag || before.Absent != after.Absent { |
| 289 | t.Fatal("refusal changed claim", err) |
| 290 | } |
| 291 | }) |
| 292 | } |
| 293 | } |
| 294 | |
amadan