| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "crypto/rand" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 9 | "encoding/json" |
| 10 | "errors" |
| 11 | "go/ast" |
| 12 | "go/parser" |
| 13 | "go/token" |
| 14 | "strings" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/domains" |
| 19 | "github.com/carlosframework/platform/internal/hostactors" |
| 20 | "github.com/carlosframework/platform/internal/hostclaim" |
| 21 | "github.com/carlosframework/platform/internal/hostops" |
| 22 | "github.com/carlosframework/platform/internal/hostwrite" |
| 23 | "github.com/carlosframework/platform/internal/provision" |
| 24 | "github.com/carlosframework/platform/internal/release" |
| 25 | ) |
| 26 | |
| 27 | func TestClaimBackfillRunsOnlyOnInstancesCadence(t *testing.T) { |
| 28 | file, err := parser.ParseFile(token.NewFileSet(), "agent.go", nil, 0) |
| 29 | if err != nil { |
| 30 | t.Fatal(err) |
| 31 | } |
| 32 | for _, decl := range file.Decls { |
| 33 | fn, ok := decl.(*ast.FuncDecl) |
| 34 | if !ok { |
| 35 | continue |
| 36 | } |
| 37 | switch fn.Name.Name { |
| 38 | case "fastConvergeLoop": |
| 39 | ast.Inspect(fn.Body, func(n ast.Node) bool { |
| 40 | call, ok := n.(*ast.CallExpr) |
| 41 | if !ok { |
| 42 | return true |
| 43 | } |
| 44 | id, ok := call.Fun.(*ast.Ident) |
| 45 | if ok && (id.Name == "reconcileBackfill" || id.Name == "newClaimBackfill") { |
| 46 | t.Error("backfill runs on the serial promote path") |
| 47 | } |
| 48 | return true |
| 49 | }) |
| 50 | case "instancesLoop": |
| 51 | constructors := 0 |
| 52 | for _, stmt := range fn.Body.List { |
| 53 | loop, inLoop := stmt.(*ast.ForStmt) |
| 54 | var node ast.Node = stmt |
| 55 | if inLoop { |
| 56 | node = loop.Body |
| 57 | } |
| 58 | ast.Inspect(node, func(n ast.Node) bool { |
| 59 | call, ok := n.(*ast.CallExpr) |
| 60 | if !ok { |
| 61 | return true |
| 62 | } |
| 63 | id, ok := call.Fun.(*ast.Ident) |
| 64 | if ok && id.Name == "newClaimBackfill" { |
| 65 | constructors++ |
| 66 | if inLoop { |
| 67 | t.Error("backfill controller constructed per tick") |
| 68 | } |
| 69 | } |
| 70 | return true |
| 71 | }) |
| 72 | } |
| 73 | if constructors != 1 { |
| 74 | t.Errorf("instances loop has %d persistent backfill constructors, want 1", constructors) |
| 75 | } |
| 76 | } |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | type backfillPassStore struct { |
| 81 | release.Store |
| 82 | gets, censusReads int |
| 83 | failClaim error |
| 84 | } |
| 85 | |
| 86 | func (s *backfillPassStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 87 | s.gets++ |
| 88 | if strings.HasPrefix(key, "control/host-census/") { |
| 89 | s.censusReads++ |
| 90 | } |
| 91 | return s.Store.Get(ctx, key) |
| 92 | } |
| 93 | |
| 94 | func (s *backfillPassStore) Put(ctx context.Context, key string, body []byte, tag string) (string, error) { |
| 95 | if strings.HasPrefix(key, "control/hosts/") && s.failClaim != nil { |
| 96 | err := s.failClaim |
| 97 | s.failClaim = nil |
| 98 | return "", err |
| 99 | } |
| 100 | return s.Store.Put(ctx, key, body, tag) |
| 101 | } |
| 102 | |
| 103 | func TestClaimBackfillPassInertAndMissingContract(t *testing.T) { |
| 104 | store := &backfillPassStore{Store: release.NewDirStore(t.TempDir())} |
| 105 | ctx := context.Background() |
| 106 | if err := newClaimBackfill(hostwrite.Config{}, store, nil, "")(ctx); err != nil || store.gets != 0 { |
| 107 | t.Fatal("flag off accessed backfill", err, store.gets) |
| 108 | } |
| 109 | for _, missing := range []string{"store", "key", "bucket"} { |
| 110 | var s release.Store = store |
| 111 | pub := make(ed25519.PublicKey, ed25519.PublicKeySize) |
| 112 | bucket := "bucket" |
| 113 | switch missing { |
| 114 | case "store": |
| 115 | s = nil |
| 116 | case "key": |
| 117 | pub = nil |
| 118 | case "bucket": |
| 119 | bucket = "" |
| 120 | } |
| 121 | if err := newClaimBackfill(hostwrite.Config{Enabled: true}, s, pub, bucket)(ctx); !errors.Is(err, hostops.ErrContract) || store.gets != 0 { |
| 122 | t.Fatalf("missing %s: err=%v reads=%d", missing, err, store.gets) |
| 123 | } |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | func TestClaimBackfillPassKeepsCensusBetweenRetries(t *testing.T) { |
| 128 | ctx := context.Background() |
| 129 | store := &backfillPassStore{Store: release.NewDirStore(t.TempDir())} |
| 130 | pub, key, err := ed25519.GenerateKey(rand.Reader) |
| 131 | if err != nil { |
| 132 | t.Fatal(err) |
| 133 | } |
| 134 | boxPub, boxKey, err := ed25519.GenerateKey(rand.Reader) |
| 135 | if err != nil { |
| 136 | t.Fatal(err) |
| 137 | } |
| 138 | now := time.Now().UTC() |
| 139 | put := func(path string, value any, signed bool) { |
| 140 | t.Helper() |
| 141 | raw, err := json.Marshal(value) |
| 142 | if signed { |
| 143 | raw, err = release.SignRootEnvelope(release.RootSigner(key), value) |
| 144 | } |
| 145 | if err != nil { |
| 146 | t.Fatal(err) |
| 147 | } |
| 148 | if _, err := store.Put(ctx, path, raw, ""); err != nil { |
| 149 | t.Fatal(err) |
| 150 | } |
| 151 | } |
| 152 | enrollment := strings.Repeat("a", 32) |
| 153 | roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, |
| 154 | Regions: map[string]hostactors.Region{"ie": {}}, |
| 155 | Boxes: map[string]hostactors.Box{"box-a": {CensusPublicKey: boxPub, Enrollment: enrollment, Namespace: "platform", BoxID: "box-a", Label: "box-a", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "ie", InstanceID: "i-box-a"}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"}}, |
| 156 | Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} |
| 157 | put(hostactors.Key, roster, true) |
| 158 | r, err := hostops.PrepareInstance(provision.Record{Schema: 1, Account: "bab", App: "jam", Host: "legacy.example.com", Slug: "shop", Channel: "edge", Region: "ie", Acquisition: "acquisition"}, now, "ie") |
| 159 | if err != nil { |
| 160 | t.Fatal(err) |
| 161 | } |
| 162 | if _, err := store.Put(ctx, r.Holder.Key, r.Body, ""); err != nil { |
| 163 | t.Fatal(err) |
| 164 | } |
| 165 | r, err = hostops.ReadRecord(ctx, store, r.Holder.Key) |
| 166 | if err != nil { |
| 167 | t.Fatal(err) |
| 168 | } |
| 169 | inc, err := r.Incarnation() |
| 170 | if err != nil { |
| 171 | t.Fatal(err) |
| 172 | } |
| 173 | put(domains.ClaimKey(r.Holder.Host), domains.Claim{Schema: 1, Host: r.Holder.Host, Account: "bab", App: "jam", Acquisition: "acquisition", RecordIncarnation: inc, OwnershipState: "committed"}, false) |
| 174 | report := hostops.CensusReport{Box: "box-a", Enrollment: enrollment, BootID: "boot", Reservation: "backfill", DirectoryRevision: 1, CapturedAt: now, Complete: true, Installations: []hostops.CensusInstallation{{Host: r.Holder.Host, Account: "bab", App: "jam", Family: "instance", Key: r.Holder.Key, Incarnation: inc, Local: true, Route: true}}} |
| 175 | canonical, err := json.Marshal(report) |
| 176 | if err != nil { |
| 177 | t.Fatal(err) |
| 178 | } |
| 179 | reportPath := "control/host-census/backfill/box-a.json" |
| 180 | put(reportPath, hostops.CensusEnvelope{Report: report, Signature: ed25519.Sign(boxKey, canonical)}, false) |
| 181 | raw, _, err := store.Get(ctx, reportPath) |
| 182 | if err != nil { |
| 183 | t.Fatal(err) |
| 184 | } |
| 185 | digest := sha256.Sum256(raw) |
| 186 | put(hostops.BackfillKey, hostops.BackfillPlan{V: 1, ID: "backfill", Phase: "adopting", DirectoryRevision: 1, CreatedAt: now, Reports: map[string]string{"box-a": hex.EncodeToString(digest[:])}, Hosts: map[string]hostops.BackfillHost{r.Holder.Host: {Key: r.Holder.Key, ETag: r.ETag, Acquisition: "acquisition", Region: "ie", Box: "box-a"}}}, true) |
| 187 | store.censusReads = 0 |
| 188 | pass := newClaimBackfill(hostwrite.Config{Enabled: true}, store, pub, "bucket") |
| 189 | fail := errors.New("claim write failed") |
| 190 | store.failClaim = fail |
| 191 | if err := pass(ctx); !errors.Is(err, fail) { |
| 192 | t.Fatalf("first pass: %v", err) |
| 193 | } |
| 194 | if err := pass(ctx); err != nil { |
| 195 | t.Fatalf("retry: %v", err) |
| 196 | } |
| 197 | if store.censusReads != 1 { |
| 198 | t.Fatalf("census reads=%d, want one across both passes", store.censusReads) |
| 199 | } |
| 200 | obs, err := hostclaim.Read(ctx, store, r.Holder.Host) |
| 201 | if err != nil || obs.Absent || obs.Claim.Holder.Acquisition != "acquisition" || obs.Claim.Op.Exec != "box-a" { |
| 202 | t.Fatalf("retry did not adopt: %+v %v", obs, err) |
| 203 | } |
| 204 | } |
| 205 | |
| 206 | func TestClaimBackfillPassQuietWithoutAdoptingPlan(t *testing.T) { |
| 207 | for _, phase := range []string{"absent", "collecting", "done", "future"} { |
| 208 | t.Run(phase, func(t *testing.T) { |
| 209 | ctx := context.Background() |
| 210 | pub, key, err := ed25519.GenerateKey(rand.Reader) |
| 211 | if err != nil { |
| 212 | t.Fatal(err) |
| 213 | } |
| 214 | store := &backfillPassStore{Store: release.NewDirStore(t.TempDir())} |
| 215 | if phase != "absent" { |
| 216 | raw, err := release.SignRootEnvelope(release.RootSigner(key), hostops.BackfillPlan{V: 1, ID: "backfill", Phase: phase, CreatedAt: time.Now()}) |
| 217 | if err != nil { |
| 218 | t.Fatal(err) |
| 219 | } |
| 220 | if _, err := store.Put(ctx, hostops.BackfillKey, raw, ""); err != nil { |
| 221 | t.Fatal(err) |
| 222 | } |
| 223 | } |
| 224 | pass := newClaimBackfill(hostwrite.Config{Enabled: true}, store, pub, "bucket") |
| 225 | if err := pass(ctx); err != nil || store.gets != 1 { |
| 226 | t.Fatalf("inactive backfill: err=%v reads=%d", err, store.gets) |
| 227 | } |
| 228 | }) |
| 229 | } |
| 230 | } |
| 231 | |
amadan