| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "crypto/ed25519" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "net/http" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/hostactors" |
| 15 | "github.com/carlosframework/platform/internal/hostfloor" |
| 16 | "github.com/carlosframework/platform/internal/release" |
| 17 | "github.com/carlosframework/platform/internal/status" |
| 18 | ) |
| 19 | |
| 20 | // plan9ActivationWorld is activationFixture's published deployment, built |
| 21 | // on an existing bucket and key so the HTTP doors can be exercised: stable |
| 22 | // points at v1.0.0 with v1.1.0 in its history, the floor is published and |
| 23 | // one active box reports v1.0.0 on stable. |
| 24 | func plan9ActivationWorld(t *testing.T, store release.Store, key ed25519.PrivateKey) release.SystemRelease { |
| 25 | t.Helper() |
| 26 | ctx := context.Background() |
| 27 | sg := release.RootSigner(key) |
| 28 | pub := sg.RootPublicKey() |
| 29 | old := release.SystemRelease{Version: "v1.0.0", Source: "release:v1.0.0", ReleaseTag: "v1.0.0", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{"linux-amd64": {Key: release.SystemArtifactKey("v1.0.0", "linux-amd64"), SHA256: strings.Repeat("a", 64)}}} |
| 30 | next := release.SystemRelease{Version: "v1.1.0", Source: "release:v1.1.0", ReleaseTag: "v1.1.0", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{"linux-amd64": {Key: release.SystemArtifactKey("v1.1.0", "linux-amd64"), SHA256: strings.Repeat("b", 64)}}} |
| 31 | for _, rel := range []release.SystemRelease{next, old} { |
| 32 | if _, err := release.PromoteSystem(ctx, store, sg, "router", "stable", rel, release.SystemPromoteOptions{}); err != nil { |
| 33 | t.Fatal(err) |
| 34 | } |
| 35 | raw, _, err := store.Get(ctx, release.SystemPointerKey("router", "stable")) |
| 36 | if err != nil { |
| 37 | t.Fatal(err) |
| 38 | } |
| 39 | if err := release.PublishSystemIdentity(ctx, store, pub, raw); err != nil { |
| 40 | t.Fatal(err) |
| 41 | } |
| 42 | } |
| 43 | if err := hostfloor.Publish(ctx, store, sg, "bucket", hostfloor.Floor{Min: 1, MinRelease: "v1.0.0"}); err != nil { |
| 44 | t.Fatal(err) |
| 45 | } |
| 46 | floor, err := hostfloor.Read(ctx, store, pub, "bucket") |
| 47 | if err != nil { |
| 48 | t.Fatal(err) |
| 49 | } |
| 50 | enrollment := strings.Repeat("e", 32) |
| 51 | roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Publication: "published", FloorSHA256: floor.Digest, Boxes: map[string]hostactors.Box{"box": {BoxID: "box", Namespace: "platform", Label: "box", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-box"}, Enrollment: enrollment, EvidenceSHA256: strings.Repeat("f", 64), State: "active"}}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} |
| 52 | raw, err := release.SignRootEnvelope(sg, roster) |
| 53 | if err != nil { |
| 54 | t.Fatal(err) |
| 55 | } |
| 56 | if _, err := store.Put(ctx, hostactors.Key, raw, ""); err != nil { |
| 57 | t.Fatal(err) |
| 58 | } |
| 59 | box := status.Box{Label: "box", GeneratedAt: time.Now(), Metrics: &status.BoxMetrics{Enrollment: enrollment, InstanceID: "i-box", Release: old.Version, ReleaseTag: old.ReleaseTag, Platform: "linux-amd64", BinarySHA256: old.Artifacts["linux-amd64"].SHA256, MovesProtocol: 1, BootID: "boot", FloorSHA256: floor.Digest, FloorGuardReady: true, HostClaimEnabled: true, SystemChannel: "stable", Layout: &status.LayoutReport{Mode: "flat", Rows: map[string]status.RowLayout{}}}} |
| 60 | raw, err = json.Marshal(box) |
| 61 | if err != nil { |
| 62 | t.Fatal(err) |
| 63 | } |
| 64 | if _, err := store.Put(ctx, "status/boxes/box.json", raw, ""); err != nil { |
| 65 | t.Fatal(err) |
| 66 | } |
| 67 | return next |
| 68 | } |
| 69 | |
| 70 | func plan9Roster(t *testing.T, s *Server) hostactors.Observed { |
| 71 | t.Helper() |
| 72 | d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: release.RootSigner(s.cfg.Key).RootPublicKey(), Bucket: s.cfg.Bucket} |
| 73 | o, err := d.Read(context.Background()) |
| 74 | if err != nil { |
| 75 | t.Fatal(err) |
| 76 | } |
| 77 | return o |
| 78 | } |
| 79 | |
| 80 | // M-8: a host update reservation is created only against a roster that is |
| 81 | // published at exactly the floor being enforced. A floor whose roster is |
| 82 | // still preparing, unpublished or bound to another digest is an unfinished |
| 83 | // publication and refuses, with no reservation and no pointer movement. |
| 84 | func TestPlan9ReserveHostUpdateRequiresPublishedRosterAtFloor(t *testing.T) { |
| 85 | for _, tc := range []string{"preparing", "unpublished", "other-digest"} { |
| 86 | t.Run(tc, func(t *testing.T) { |
| 87 | ctx := context.Background() |
| 88 | s, d, target := activationFixture(t) |
| 89 | before := plan9Roster(t, s) |
| 90 | r := hostactors.Clone(before.Roster) |
| 91 | switch tc { |
| 92 | case "preparing": |
| 93 | r.Publication = "preparing" |
| 94 | case "unpublished": |
| 95 | r.Publication, r.FloorSHA256 = "", "" |
| 96 | case "other-digest": |
| 97 | r.FloorSHA256 = strings.Repeat("0", 64) |
| 98 | } |
| 99 | raw, err := release.SignRootEnvelope(release.RootSigner(s.cfg.Key), r) |
| 100 | if err != nil { |
| 101 | t.Fatal(err) |
| 102 | } |
| 103 | if _, err := d.Store.Put(ctx, hostactors.Key, raw, before.ETag); err != nil { |
| 104 | t.Fatal(err) |
| 105 | } |
| 106 | pointer, _, err := d.Store.Get(ctx, release.SystemPointerKey("router", "stable")) |
| 107 | if err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | if id, err := s.reserveHostUpdate(ctx, &target, "stable", "promote", release.SystemPromoteOptions{}); err == nil || id != "" { |
| 111 | t.Fatalf("%s: reservation = %q, %v; want refusal", tc, id, err) |
| 112 | } |
| 113 | if _, err := s.promoteHostUpdate(ctx, target, "router", "stable", release.SystemPromoteOptions{}); err == nil { |
| 114 | t.Fatalf("%s: promote published against an unfinished floor publication", tc) |
| 115 | } |
| 116 | after := plan9Roster(t, s) |
| 117 | if len(after.Roster.Activations) != 0 { |
| 118 | t.Fatalf("%s: reservation written", tc) |
| 119 | } |
| 120 | now, _, err := d.Store.Get(ctx, release.SystemPointerKey("router", "stable")) |
| 121 | if err != nil || !bytes.Equal(now, pointer) { |
| 122 | t.Fatalf("%s: pointer moved: %v", tc, err) |
| 123 | } |
| 124 | }) |
| 125 | } |
| 126 | } |
| 127 | |
| 128 | // casOncePointerStore answers one conditional pointer write with the |
| 129 | // conflict S3 gives when another writer moved the pointer first. |
| 130 | type casOncePointerStore struct { |
| 131 | release.Store |
| 132 | puts int |
| 133 | } |
| 134 | |
| 135 | func (s *casOncePointerStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) { |
| 136 | if key == release.SystemPointerKey("router", "canary") { |
| 137 | s.puts++ |
| 138 | return "", release.ErrCASConflict |
| 139 | } |
| 140 | return s.Store.Put(ctx, key, raw, etag) |
| 141 | } |
| 142 | |
| 143 | // M-6: with the flag off and no floor, a rollback that loses its pointer |
| 144 | // CAS answers 409 and does not retry. A retry would pop history again |
| 145 | // against a pointer the operator never saw. |
| 146 | func TestPlan9RollbackCASConflictIsAConflictAndNotRetried(t *testing.T) { |
| 147 | fx := newConsoleFixture(t) |
| 148 | owner := cliToken(t, fx, "paul@keymail.dev", "code-plan9-rollback") |
| 149 | sg := release.RootSigner(fx.priv) |
| 150 | rel := func(v string) release.SystemRelease { |
| 151 | return release.SystemRelease{Version: v, Artifacts: map[string]release.SystemArtifact{"linux-arm64": {Key: release.SystemArtifactKey(v, "linux-arm64"), SHA256: strings.Repeat("a", 64)}}} |
| 152 | } |
| 153 | if _, err := release.PromoteSystem(context.Background(), fx.rel, sg, "router", "canary", rel("v1.25.0"), release.SystemPromoteOptions{}); err != nil { |
| 154 | t.Fatal(err) |
| 155 | } |
| 156 | time.Sleep(1100 * time.Millisecond) |
| 157 | if _, err := release.PromoteSystem(context.Background(), fx.rel, sg, "router", "canary", rel("v1.26.0"), release.SystemPromoteOptions{}); err != nil { |
| 158 | t.Fatal(err) |
| 159 | } |
| 160 | pointer, _, err := fx.rel.Get(context.Background(), release.SystemPointerKey("router", "canary")) |
| 161 | if err != nil { |
| 162 | t.Fatal(err) |
| 163 | } |
| 164 | wrapped := &casOncePointerStore{Store: fx.rel} |
| 165 | fx.srv.cfg.Releases = wrapped |
| 166 | st, out := postJSON(t, fx.ts.URL+"/api/cli/system/rollback", map[string]string{"channel": "canary"}, owner) |
| 167 | if st != http.StatusConflict { |
| 168 | t.Fatalf("rollback after a lost CAS = %d %v, want 409", st, out) |
| 169 | } |
| 170 | if msg, _ := out["error"].(string); strings.Contains(msg, "compare-and-swap") || strings.Contains(msg, "—") { |
| 171 | t.Fatalf("409 body leaks the store error or uses an em dash: %q", msg) |
| 172 | } |
| 173 | if wrapped.puts != 1 { |
| 174 | t.Fatalf("pointer writes = %d, want exactly one attempt", wrapped.puts) |
| 175 | } |
| 176 | now, _, err := fx.rel.Get(context.Background(), release.SystemPointerKey("router", "canary")) |
| 177 | if err != nil || !bytes.Equal(now, pointer) { |
| 178 | t.Fatalf("pointer changed: %v", err) |
| 179 | } |
| 180 | } |
| 181 | |
| 182 | // I-4: an unpublished activation reservation blocks rollback and every |
| 183 | // other promote of its channel. An ops owner or system operator, on a person |
| 184 | // session, cancels it: the pointer is fenced at the publication's expected |
| 185 | // ETag first, so a delayed publication can never land, and only then is |
| 186 | // the reservation removed. Rollback and promote work again afterwards. |
| 187 | func TestPlan9OperatorCancelsAnUnpublishedActivation(t *testing.T) { |
| 188 | fx := newConsoleFixture(t) |
| 189 | owner := cliToken(t, fx, "paul@keymail.dev", "code-plan9-cancel") |
| 190 | fx.srv.cfg.Bucket = "bucket" |
| 191 | target := plan9ActivationWorld(t, fx.rel, fx.priv) |
| 192 | ctx := context.Background() |
| 193 | id, err := fx.srv.reserveHostUpdate(ctx, &target, "stable", "promote", release.SystemPromoteOptions{}) |
| 194 | if err != nil || id == "" { |
| 195 | t.Fatal(id, err) |
| 196 | } |
| 197 | held := plan9Roster(t, fx.srv) |
| 198 | stale := *held.Roster.Activations[id].Publication |
| 199 | // The wedge: a different request for the channel is refused while held. |
| 200 | if _, err := fx.srv.rollbackHostUpdate(ctx, "router", "stable", "paul@keymail.dev"); err == nil { |
| 201 | t.Fatal("rollback succeeded through a held activation") |
| 202 | } |
| 203 | if fx.srv.cfg.Releases == nil { |
| 204 | t.Fatal("fixture has no bucket") |
| 205 | } |
| 206 | if fx.st.AddMember(OpsAccountID, "member@keymail.dev", RoleMember, time.Now()) != nil { |
| 207 | t.Fatal("add member") |
| 208 | } |
| 209 | member := cliToken(t, fx, "member@keymail.dev", "code-plan9-cancel-member") |
| 210 | _, want := getRaw(t, fx.ts.URL+"/api/cli/system/no-such-door", member) |
| 211 | if st, body := postRaw(t, fx.ts.URL+"/api/cli/system/activation/cancel", map[string]string{"channel": "stable"}, member); st != http.StatusNotFound || body != want { |
| 212 | t.Fatalf("member cancel = %d %q, want the bare 404", st, body) |
| 213 | } |
| 214 | if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/activation/cancel", map[string]string{"channel": "stable", "activation": strings.Repeat("0", 32)}, owner); st != http.StatusConflict { |
| 215 | t.Fatalf("cancel naming another activation = %d %v, want 409", st, out) |
| 216 | } |
| 217 | if after := plan9Roster(t, fx.srv); len(after.Roster.Activations) != 1 { |
| 218 | t.Fatal("mismatched cancel removed the reservation") |
| 219 | } |
| 220 | for _, body := range []map[string]string{{"channel": "edge"}, {"channel": ""}, {"channel": "stable", "component": "../etc"}} { |
| 221 | if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/activation/cancel", body, owner); st != http.StatusBadRequest { |
| 222 | t.Errorf("cancel %v = %d %v, want 400", body, st, out) |
| 223 | } |
| 224 | } |
| 225 | st, out := postJSON(t, fx.ts.URL+"/api/cli/system/activation/cancel", map[string]string{"channel": "stable", "activation": id}, owner) |
| 226 | if st != http.StatusOK || str(t, out, "activation") != id { |
| 227 | t.Fatalf("cancel = %d %v", st, out) |
| 228 | } |
| 229 | if after := plan9Roster(t, fx.srv); len(after.Roster.Activations) != 0 { |
| 230 | t.Fatal("cancel left the reservation held") |
| 231 | } |
| 232 | if _, err := release.PublishSystemPublication(ctx, fx.rel, fx.pub, stale); !errors.Is(err, release.ErrCASConflict) { |
| 233 | t.Fatalf("delayed publication after cancel: %v", err) |
| 234 | } |
| 235 | if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/activation/cancel", map[string]string{"channel": "stable"}, owner); st != http.StatusNotFound { |
| 236 | t.Fatalf("cancel with nothing held = %d %v, want 404", st, out) |
| 237 | } |
| 238 | p, err := fx.srv.rollbackHostUpdate(ctx, "router", "stable", "paul@keymail.dev") |
| 239 | if err != nil || p.Current.Version != "v1.1.0" { |
| 240 | t.Fatalf("rollback after cancel = %v, %v", p, err) |
| 241 | } |
| 242 | } |
| 243 | |
| 244 | // I-4: cancellation never reaches past publication. A published activation |
| 245 | // is released only by completion or the D7 exclusions, so the door answers |
| 246 | // 409 and the reservation stays held. A read-only console cannot cancel. |
| 247 | func TestPlan9CancelRefusesAPublishedActivationAndReadOnlyConsole(t *testing.T) { |
| 248 | fx := newConsoleFixture(t) |
| 249 | owner := cliToken(t, fx, "paul@keymail.dev", "code-plan9-published") |
| 250 | fx.srv.cfg.Bucket = "bucket" |
| 251 | target := plan9ActivationWorld(t, fx.rel, fx.priv) |
| 252 | p, err := fx.srv.promoteHostUpdate(context.Background(), target, "router", "stable", release.SystemPromoteOptions{}) |
| 253 | if err != nil || p.ActivationID == "" { |
| 254 | t.Fatal(p, err) |
| 255 | } |
| 256 | st, out := postJSON(t, fx.ts.URL+"/api/cli/system/activation/cancel", map[string]string{"channel": "stable"}, owner) |
| 257 | if st != http.StatusConflict { |
| 258 | t.Fatalf("cancel of a published activation = %d %v, want 409", st, out) |
| 259 | } |
| 260 | if msg, _ := out["error"].(string); strings.Contains(msg, "—") || strings.Contains(msg, "compare-and-swap") { |
| 261 | t.Fatalf("409 body: %q", msg) |
| 262 | } |
| 263 | if held := plan9Roster(t, fx.srv); len(held.Roster.Activations) != 1 { |
| 264 | t.Fatal("published activation released by cancel") |
| 265 | } |
| 266 | fx.srv.cfg.Key = nil |
| 267 | if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/activation/cancel", map[string]string{"channel": "stable"}, owner); st != http.StatusServiceUnavailable { |
| 268 | t.Fatalf("read-only cancel = %d %v, want 503", st, out) |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | // M-3: every fresh foreign-process report refuses replacement. |
| 273 | func TestPlan9ConsoleReportRefusesAFreshForeignProcess(t *testing.T) { |
| 274 | started := time.Now().Add(-time.Minute) |
| 275 | for _, tc := range []struct { |
| 276 | name string |
| 277 | existing func() []byte |
| 278 | replaced bool |
| 279 | }{ |
| 280 | {"absent", nil, true}, |
| 281 | {"same-process", func() []byte { return plan9Report("boot:1", time.Now()) }, true}, |
| 282 | {"foreign-after-start", func() []byte { return plan9Report("boot:2", time.Now()) }, false}, |
| 283 | {"foreign-before-start", func() []byte { return plan9Report("boot:2", started.Add(-time.Second)) }, false}, |
| 284 | {"foreign-stale", func() []byte { return plan9Report("boot:2", time.Now().Add(-16*time.Minute)) }, true}, |
| 285 | {"foreign-future", func() []byte { return plan9Report("boot:2", time.Now().Add(time.Hour)) }, true}, |
| 286 | {"malformed", func() []byte { return []byte("{") }, true}, |
| 287 | } { |
| 288 | t.Run(tc.name, func(t *testing.T) { |
| 289 | ctx := t.Context() |
| 290 | store := s3ShapedStore{release.NewDirStore(t.TempDir())} |
| 291 | const key = "status/consoles/console-2.json" |
| 292 | if tc.existing != nil { |
| 293 | if _, err := store.Put(ctx, key, tc.existing(), ""); err != nil { |
| 294 | t.Fatal(err) |
| 295 | } |
| 296 | } |
| 297 | mine := plan9Report("boot:1", time.Now()) |
| 298 | err := putHostReport(ctx, store, key, mine) |
| 299 | got, _, readErr := store.Get(ctx, key) |
| 300 | if readErr != nil { |
| 301 | t.Fatal(readErr) |
| 302 | } |
| 303 | if tc.replaced { |
| 304 | if err != nil || !bytes.Equal(got, mine) { |
| 305 | t.Fatalf("%s: put = %v; want replaced", tc.name, err) |
| 306 | } |
| 307 | return |
| 308 | } |
| 309 | if err == nil || bytes.Equal(got, mine) { |
| 310 | t.Fatalf("%s: put = %v; want refusal and the foreign report kept", tc.name, err) |
| 311 | } |
| 312 | if strings.Contains(err.Error(), "—") { |
| 313 | t.Fatalf("error uses an em dash: %v", err) |
| 314 | } |
| 315 | }) |
| 316 | } |
| 317 | } |
| 318 | |
| 319 | func plan9Report(process string, at time.Time) []byte { |
| 320 | raw, _ := json.Marshal(hostactors.Report{Component: "console", Actor: "console-2", ProcessID: process, GeneratedAt: at.UTC()}) |
| 321 | return raw |
| 322 | } |
| 323 | |
amadan