| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "encoding/json" |
| 7 | "github.com/carlosframework/platform/internal/hostclaim" |
| 8 | "github.com/carlosframework/platform/internal/hostguard" |
| 9 | "github.com/carlosframework/platform/internal/provision" |
| 10 | "github.com/carlosframework/platform/internal/registry" |
| 11 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 12 | "github.com/carlosframework/platform/internal/rootstate" |
| 13 | "os" |
| 14 | "path/filepath" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/release" |
| 19 | "github.com/carlosframework/platform/internal/rollout" |
| 20 | ) |
| 21 | |
| 22 | func TestPlan14SwapFailureSurvivesLegacyTampering(t *testing.T) { |
| 23 | for _, attack := range []string{"erase", "rename", "retried", "epoch", "served"} { |
| 24 | t.Run(attack, func(t *testing.T) { |
| 25 | s := testRootFailureStore(t) |
| 26 | b, err := rollout.InitializeRootFailures(s, "box/en/1", nil) |
| 27 | if err != nil { |
| 28 | t.Fatal(err) |
| 29 | } |
| 30 | _, err = b.Record(rollout.FailureRecord{Account: "bab", App: "km", Build: "v1", Host: "h.example.com", HeldDown: []string{"h.example.com"}}) |
| 31 | if err != nil { |
| 32 | t.Fatal(err) |
| 33 | } |
| 34 | dd := t.TempDir() |
| 35 | raw := []byte(`{"schema":3,"records":[{"account":"bab","app":"km","build":"v1","host":"h.example.com","retried":["h.example.com"],"epochs":{"h.example.com":1}}]}`) |
| 36 | if err := os.WriteFile(filepath.Join(dd, rollout.FailureFile), raw, 0600); err != nil { |
| 37 | t.Fatal(err) |
| 38 | } |
| 39 | if attack == "erase" { |
| 40 | if err := os.Remove(filepath.Join(dd, rollout.FailureFile)); err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | } |
| 44 | if attack == "rename" { |
| 45 | if err := os.Rename(dd, dd+"-old"); err != nil { |
| 46 | t.Fatal(err) |
| 47 | } |
| 48 | t.Cleanup(func() { os.RemoveAll(dd + "-old") }) |
| 49 | } |
| 50 | legacy, err := rootFailureLegacy(dd) |
| 51 | if err != nil { |
| 52 | t.Fatal(err) |
| 53 | } |
| 54 | if err = b.ImportLegacy(legacy); err != nil { |
| 55 | t.Fatal(err) |
| 56 | } |
| 57 | b, err = rollout.OpenRootFailures(s, "box/en/1") |
| 58 | if err != nil { |
| 59 | t.Fatal(err) |
| 60 | } |
| 61 | if !b.Blocks("bab", "km", "v1", "sibling.example.com", false) || !b.HeldDown("bab", "km", "v1", "h.example.com") { |
| 62 | t.Fatal("root failure lost") |
| 63 | } |
| 64 | testRootSettingsPass(t, b, attack) |
| 65 | }) |
| 66 | } |
| 67 | } |
| 68 | |
| 69 | func TestPlan14SwapFailureVerifiedRetry(t *testing.T) { |
| 70 | s := testRootFailureStore(t) |
| 71 | b, err := rollout.InitializeRootFailures(s, "box/en/1", nil) |
| 72 | if err != nil { |
| 73 | t.Fatal(err) |
| 74 | } |
| 75 | rec := rollout.FailureRecord{Account: "bab", App: "km", Build: "v1", Host: "h.example.com", HeldDown: []string{"h.example.com"}} |
| 76 | if _, err = b.Record(rec); err != nil { |
| 77 | t.Fatal(err) |
| 78 | } |
| 79 | seed, public, _ := release.GenerateKey() |
| 80 | priv, _ := release.ParseSigningKey(seed, "test") |
| 81 | pub, _ := release.ParsePublicKey(public, "test") |
| 82 | wire, err := release.SignHostRetry(release.RootSigner(priv), "bab", "km", rec.Host, "v1", b.ID(), b.PublishedEpoch("bab", "km", "v1", rec.Host), time.Now()) |
| 83 | if err != nil { |
| 84 | t.Fatal(err) |
| 85 | } |
| 86 | if err = b.VerifiedRetry(pub, wire, "bab", "km", rec.Host, time.Now()); err != nil { |
| 87 | t.Fatal(err) |
| 88 | } |
| 89 | if b.HeldDown("bab", "km", "v1", rec.Host) || b.Blocks("bab", "km", "v1", rec.Host, false) { |
| 90 | t.Fatal("exact retry did not clear") |
| 91 | } |
| 92 | if !b.Blocks("bab", "km", "v1", "sibling.example.com", false) { |
| 93 | t.Fatal("retry widened") |
| 94 | } |
| 95 | if _, err = b.Record(rec); err != nil { |
| 96 | t.Fatal(err) |
| 97 | } |
| 98 | if err = b.VerifiedRetry(pub, wire, "bab", "km", rec.Host, time.Now()); err != nil { |
| 99 | t.Fatal(err) |
| 100 | } |
| 101 | if !b.HeldDown("bab", "km", "v1", rec.Host) { |
| 102 | t.Fatal("stale retry cleared second failure") |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | func TestPlan14SwapFailureMigrationUnknown(t *testing.T) { |
| 107 | s := testRootFailureStore(t) |
| 108 | if _, err := rollout.OpenRootFailures(s, "box/en/1"); err == nil { |
| 109 | t.Fatal("absent marker granted empty history") |
| 110 | } |
| 111 | b, err := rollout.InitializeRootFailures(s, "box/en/1", nil) |
| 112 | if err != nil { |
| 113 | t.Fatal(err) |
| 114 | } |
| 115 | if b.Blocks("bab", "km", "v2", "h.example.com", false) { |
| 116 | t.Fatal("unknown history refused an unfailed signed target") |
| 117 | } |
| 118 | if _, err := rollout.OpenRootFailures(s, "other/en/1"); err == nil { |
| 119 | t.Fatal("foreign migration accepted") |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | func testRootSettingsPass(t *testing.T, book *rollout.RootFailures, attack string, activation ...bool) { |
| 124 | t.Helper() |
| 125 | ctx := context.Background() |
| 126 | opt := t.TempDir() |
| 127 | store := release.NewDirStore(t.TempDir()) |
| 128 | seed, public, _ := release.GenerateKey() |
| 129 | priv, _ := release.ParseSigningKey(seed, "test") |
| 130 | pub, _ := release.ParsePublicKey(public, "test") |
| 131 | for _, v := range []string{"v1", "v2"} { |
| 132 | m := release.Manifest{Schema: 1, App: "km", Version: v, Kind: "binary", Artifacts: map[string]release.Artifact{runtimePlatform(): {}}} |
| 133 | if err := release.Ship(ctx, store, "bab", m, map[string][]byte{runtimePlatform(): []byte(v)}); err != nil { |
| 134 | t.Fatal(err) |
| 135 | } |
| 136 | if err := release.Promote(ctx, store, priv, "bab", "km", "canary/x", v); err != nil { |
| 137 | t.Fatal(err) |
| 138 | } |
| 139 | } |
| 140 | rec := provision.Record{Schema: 1, Host: "h.example.com", Account: "bab", App: "km", Channel: "canary/x", Pin: "v1", CreatedAt: time.Now().UTC()} |
| 141 | body, _ := json.Marshal(rec) |
| 142 | if _, err := store.Put(ctx, provision.RecordKey(rec.Account, rec.App, rec.Host), body, ""); err != nil { |
| 143 | t.Fatal(err) |
| 144 | } |
| 145 | inc, _ := hostclaim.Incarnation(rec.Account, rec.App, rec.Host, rec.CreatedAt) |
| 146 | state := &rollbackMemory{files: map[string][]byte{}} |
| 147 | in := hostguard.RootInstallation{Host: rec.Host, Incarnation: inc, Stamp: 1, Account: rec.Account, App: rec.App, Record: provision.RecordKey(rec.Account, rec.App, rec.Host), Box: "box", Enrollment: "en", Migration: 1, InventoryRevision: 1} |
| 148 | if err := hostguard.NewRootAuthority(state).RetainMigrationDeny(ctx, in, 1); err != nil { |
| 149 | t.Fatal(err) |
| 150 | } |
| 151 | census := func() (rootparticipation.Census, error) { |
| 152 | return rootparticipation.Census{Schema: 1, Box: "box", Enrollment: "en", Migration: 1, Complete: true}, nil |
| 153 | } |
| 154 | p := rootUpdateSource{enforceFailures: len(activation) == 0, source: rootRestartRequestSource{state: state, store: store, census: census}, state: state, book: book, store: store, pub: pub, binding: &proofBindingSource{}, data: t.TempDir()} |
| 155 | if len(activation) != 0 { |
| 156 | p.publish = func(string, rootRetryStatusRecord) error { return os.ErrPermission } |
| 157 | } |
| 158 | rt := provision.RouteFor(rec, p.data, "", false, nil) |
| 159 | reg, err := registry.Open(filepath.Join(t.TempDir(), "r.db")) |
| 160 | if err != nil { |
| 161 | t.Fatal(err) |
| 162 | } |
| 163 | defer reg.Close() |
| 164 | switch attack { |
| 165 | case "empty", "erase", "rename": |
| 166 | rt.UpdateSettings = "" |
| 167 | case "object": |
| 168 | rt.UpdateSettings = "{}" |
| 169 | case "channel": |
| 170 | rt.UpdateSettings = `{"updates":"auto","channel":"canary/forged"}` |
| 171 | rt.Channel = "canary/forged" |
| 172 | case "served", "retried", "epoch": |
| 173 | rt.GoodVersion = "v1" |
| 174 | case "digest": |
| 175 | rt.GoodDigest = "forged" |
| 176 | rt.GoodDemand = "forged" |
| 177 | rt.ServingVersion = "v1" |
| 178 | case "build": |
| 179 | rt.BuildState = release.BuildStateCopy{Known: true, State: &release.BuildState{LastExecuted: "v1", Completed: 99}}.Column() |
| 180 | case "combined": |
| 181 | rt.UpdateSettings = "" |
| 182 | rt.GoodVersion = "v1" |
| 183 | rt.BuildState = release.BuildStateCopy{Known: true}.Column() |
| 184 | } |
| 185 | if err = reg.Upsert(rt); err != nil { |
| 186 | t.Fatal(err) |
| 187 | } |
| 188 | provider := func(context.Context, release.Store, ed25519.PublicKey) (*release.RootUpdateInputs, error) { |
| 189 | return &release.RootUpdateInputs{Resolve: p.resolve}, nil |
| 190 | } |
| 191 | effects := 0 |
| 192 | pass := func() error { |
| 193 | return release.Pass(ctx, reg, store, pub, opt, func(string) error { effects++; return nil }, updatesPassOption(p.data, true, provider)) |
| 194 | } |
| 195 | if err = pass(); err != nil { |
| 196 | t.Fatal(err) |
| 197 | } |
| 198 | want := "" |
| 199 | if len(activation) != 0 { |
| 200 | want = "v1" |
| 201 | } |
| 202 | if v, _ := release.LiveVersion(opt, rt.Host); v != want || effects != 0 { |
| 203 | t.Fatal("wrong activation failure policy", v, want, effects) |
| 204 | } |
| 205 | if len(activation) != 0 { |
| 206 | if !book.HeldDown(rec.Account, rec.App, "v1", rec.Host) { |
| 207 | t.Fatal("attribution erased") |
| 208 | } |
| 209 | if saved, err := p.resolve(ctx, rec.Host); err != nil || saved.Failed("v1") || saved.Failed("v2") { |
| 210 | t.Fatal("sticky deny active", err) |
| 211 | } |
| 212 | return |
| 213 | } |
| 214 | raw, _ := state.Read(rootSettingsKey(rt.Host, rootHistoryInstallation(in))) |
| 215 | var saved rootUpdateRecord |
| 216 | if json.Unmarshal(raw, &saved) != nil || saved.Consumed == nil || saved.Consumed.Desired != "" || saved.Consumed.Completed != 0 { |
| 217 | t.Fatal("forged completion", string(raw)) |
| 218 | } |
| 219 | |
| 220 | if attack == "verified-retry" { |
| 221 | legacy, err := rollout.OpenFailureBook(p.data) |
| 222 | if err != nil { |
| 223 | t.Fatal(err) |
| 224 | } |
| 225 | if _, err = legacy.Record(rollout.FailureRecord{Account: rec.Account, App: rec.App, Host: rec.Host, Build: "v1", HeldDown: []string{rec.Host}}); err != nil { |
| 226 | t.Fatal(err) |
| 227 | } |
| 228 | p.legacy, err = rootFailureLegacy(p.data) |
| 229 | if err != nil { |
| 230 | t.Fatal(err) |
| 231 | } |
| 232 | wire, err := release.SignHostRetry(release.RootSigner(priv), rec.Account, rec.App, rec.Host, "v1", book.ID(), book.PublishedEpoch(rec.Account, rec.App, "v1", rec.Host), time.Now()) |
| 233 | if err != nil { |
| 234 | t.Fatal(err) |
| 235 | } |
| 236 | log := release.ScopedRestartGeneration{Schema: 1, Requests: []release.RestartRequest{{ID: "root-retry", RequestedAt: time.Now().UTC().Format(time.RFC3339), Scope: release.RestartScope{Kind: release.RestartScopeRetry, Value: rec.Host, Signed: wire}}}} |
| 237 | raw, _ := json.Marshal(log) |
| 238 | if _, err = store.Put(ctx, release.HostRetryKey(rec.Account, rec.App), raw, ""); err != nil { |
| 239 | t.Fatal(err) |
| 240 | } |
| 241 | if err = pass(); err != nil { |
| 242 | t.Fatal(err) |
| 243 | } |
| 244 | if v, _ := release.LiveVersion(opt, rt.Host); v != "v1" { |
| 245 | t.Fatal("verified retry did not reach production adoption", v) |
| 246 | } |
| 247 | raw, _ = state.Read(rootSettingsKey(rt.Host, rootHistoryInstallation(in))) |
| 248 | var mirror rootUpdateRecord |
| 249 | if json.Unmarshal(raw, &mirror) != nil || mirror.RetryMirror == nil || mirror.RetryMirror.Book != legacy.ID() || mirror.RetryMirror.Epoch != legacy.PublishedEpoch(rec.Account, rec.App, "v1", rec.Host) { |
| 250 | t.Fatal("root did not bind agent mirror receipt", string(raw)) |
| 251 | } |
| 252 | |
| 253 | } |
| 254 | if attack == "provider-missing" { |
| 255 | if err := state.Remove(rec.Host + ".json"); err != nil { |
| 256 | t.Fatal(err) |
| 257 | } |
| 258 | if err := pass(); err == nil { |
| 259 | t.Fatal("missing root installation fell through") |
| 260 | } |
| 261 | return |
| 262 | } |
| 263 | // A different signed build remains available with unknown history. |
| 264 | rec.Pin = "v2" |
| 265 | body, _ = json.Marshal(rec) |
| 266 | _, etag, _ := store.Get(ctx, provision.RecordKey(rec.Account, rec.App, rec.Host)) |
| 267 | if _, err = store.Put(ctx, provision.RecordKey(rec.Account, rec.App, rec.Host), body, etag); err != nil { |
| 268 | t.Fatal(err) |
| 269 | } |
| 270 | if err = pass(); err != nil { |
| 271 | t.Fatal(err) |
| 272 | } |
| 273 | if v, _ := release.LiveVersion(opt, rt.Host); v != "v2" { |
| 274 | t.Fatal("signed positive refused", v) |
| 275 | } |
| 276 | } |
| 277 | |
| 278 | func TestPlan14RootSettingsExactRegistryMutations(t *testing.T) { |
| 279 | for _, attack := range []string{"empty", "object", "channel", "served", "digest", "build", "combined", "verified-retry", "provider-missing"} { |
| 280 | t.Run(attack, func(t *testing.T) { |
| 281 | s := testRootFailureStore(t) |
| 282 | b, err := rollout.InitializeRootFailures(s, "box/en/1", nil) |
| 283 | if err != nil { |
| 284 | t.Fatal(err) |
| 285 | } |
| 286 | record := rollout.FailureRecord{Account: "bab", App: "km", Build: "v1", Host: "h.example.com", HeldDown: []string{"h.example.com"}} |
| 287 | if attack == "served" || attack == "digest" || attack == "build" { |
| 288 | record.HeldDown = nil |
| 289 | } |
| 290 | if _, err = b.Record(record); err != nil { |
| 291 | t.Fatal(err) |
| 292 | } |
| 293 | testRootSettingsPass(t, b, attack) |
| 294 | }) |
| 295 | } |
| 296 | } |
| 297 | |
| 298 | func testRootFailureStore(t *testing.T) rootstate.Store { |
| 299 | t.Helper() |
| 300 | if os.Geteuid() != 0 { |
| 301 | return &rollbackMemory{files: map[string][]byte{}} |
| 302 | } |
| 303 | dir, err := os.MkdirTemp("/var/lib", "carlos-e3-book-") |
| 304 | if err != nil { |
| 305 | t.Fatal(err) |
| 306 | } |
| 307 | t.Cleanup(func() { os.RemoveAll(dir) }) |
| 308 | s, err := rootstate.Open(dir, false, false) |
| 309 | if err != nil { |
| 310 | t.Fatal(err) |
| 311 | } |
| 312 | t.Cleanup(func() { s.Close() }) |
| 313 | return s |
| 314 | } |
| 315 | |
| 316 | func TestPlan14RootFailureActivationGate(t *testing.T) { |
| 317 | for _, attack := range []string{"combined", "served", "channel", "build"} { |
| 318 | t.Run(attack, func(t *testing.T) { |
| 319 | b, err := rollout.InitializeRootFailures(testRootFailureStore(t), "box/en/1", nil) |
| 320 | if err != nil { |
| 321 | t.Fatal(err) |
| 322 | } |
| 323 | if _, err = b.Record(rollout.FailureRecord{Account: "bab", App: "km", Build: "v1", Host: "h.example.com", HeldDown: []string{"h.example.com"}}); err != nil { |
| 324 | t.Fatal(err) |
| 325 | } |
| 326 | if err = b.MarkHoldRan("h.example.com", rollout.HoldRan{Account: "bab", App: "km", Build: "v1", Exec: "unsettled"}); err != nil { |
| 327 | t.Fatal(err) |
| 328 | } |
| 329 | testRootSettingsPass(t, b, attack, true) |
| 330 | }) |
| 331 | } |
| 332 | } |
| 333 | |