| 1 | package release |
| 2 | |
| 3 | import ( |
| 4 | "go/ast" |
| 5 | "go/parser" |
| 6 | "go/token" |
| 7 | "io/fs" |
| 8 | "os" |
| 9 | "path/filepath" |
| 10 | "sort" |
| 11 | "strconv" |
| 12 | "strings" |
| 13 | "testing" |
| 14 | ) |
| 15 | |
| 16 | // The change index (control/changes.json) only moves if every writer of the |
| 17 | // deployment bucket bumps it. A bump lives in the concrete store and is off |
| 18 | // until EnableChangeIndex, so the way a deployment-bucket writer goes missing |
| 19 | // is a new call site that opens the bucket with the plain constructor. This |
| 20 | // test finds every place the module builds a store over a bucket or a |
| 21 | // directory (NewS3Store, NewS3StoreIn, an S3Store literal, NewDirStore) and |
| 22 | // makes each one say which kind it is for: |
| 23 | // |
| 24 | // - an S3 deployment bucket (apps/, accounts/, control/): the site calls |
| 25 | // NewDeploymentS3Store[In], so it never appears here at all; |
| 26 | // - a deployment directory (CARLOS_DEPLOYMENT_DIR) or a hand-built |
| 27 | // S3Store literal over the deployment bucket: listed in |
| 28 | // deploymentManualSites, and the test checks the same function calls |
| 29 | // EnableChangeIndex at least once per site; |
| 30 | // - anything else: listed in otherBucketOpeners with the reason. |
| 31 | // |
| 32 | // Keys are "path/from/module/root.go:enclosingFunc". Every entry carries the |
| 33 | // number of sites it covers, and the test fails when the count found under |
| 34 | // the key differs. A new open inside a listed function is therefore a |
| 35 | // failure, not a silent pass. |
| 36 | |
| 37 | type openerEntry struct { |
| 38 | reason string |
| 39 | count int |
| 40 | } |
| 41 | |
| 42 | // otherBucketOpeners are the sites that open a store which is not the |
| 43 | // deployment bucket: through NewS3Store, NewS3StoreIn, an S3Store literal or |
| 44 | // NewDirStore. |
| 45 | var otherBucketOpeners = map[string]openerEntry{ |
| 46 | "cmd/console/main.go:serve": {"console seed data store: the database/data bucket (seeds/), not the deployment bucket", 1}, |
| 47 | "internal/host/seedwiring.go:seedDataOpener": {"this region's data bucket, where seeds/ lives", 1}, |
| 48 | "internal/host/relocatepass.go:productionRelocationRuntimeInputs": {"relocation files store: the data bucket, manifests under moves/ prefixes", 1}, |
| 49 | "internal/host/relocatepass.go:registerRelocationSource": {"relocation files store: the data bucket, manifests under moves/ prefixes", 1}, |
| 50 | "internal/host/claimfiles.go:newProductionFilesStore": {"claim files manifests: the data bucket, moves/ prefixes", 1}, |
| 51 | "internal/hostfence/writer.go:OpenRootFenceStore": {"fence writer: the fence bucket, written under its own scoped role", 1}, |
| 52 | "internal/lease/execution_store.go:ExecutionStore": {"execution/build state: the hibernate and claim data bucket, keyed by host prefix", 1}, |
| 53 | "internal/status/dryrun.go:DryRun": {"status dry run: a throwaway scratch directory, not a deployment", 1}, |
| 54 | } |
| 55 | |
| 56 | // deploymentManualSites are stores over the deployment (CARLOS_DEPLOYMENT_DIR |
| 57 | // directory, or a hand-built S3Store literal over the bucket) that are not |
| 58 | // built by NewDeploymentS3Store. Each function must call EnableChangeIndex |
| 59 | // once per site. |
| 60 | var deploymentManualSites = map[string]openerEntry{ |
| 61 | "cli.go:deploymentReadSeam": {"CARLOS_DEPLOYMENT_DIR", 1}, |
| 62 | "cli.go:openStore": {"CARLOS_DEPLOYMENT_DIR", 1}, |
| 63 | "cmd/console/main.go:openStore": {"CARLOS_DEPLOYMENT_DIR", 1}, |
| 64 | "internal/edge/edge.go:enableStaticFromEnv": {"CARLOS_DEPLOYMENT_DIR", 1}, |
| 65 | "internal/host/agent.go:openDeployStoreEnv": {"CARLOS_DEPLOYMENT_DIR", 1}, |
| 66 | "internal/ops/ops.go:openDeployStore": {"CARLOS_DEPLOYMENT_DIR", 1}, |
| 67 | "internal/vet/vet.go:openStore": {"CARLOS_DEPLOYMENT_DIR", 1}, |
| 68 | } |
| 69 | |
| 70 | type openerSite struct { |
| 71 | key string // path:func |
| 72 | line int |
| 73 | what string // what was found, for the failure message |
| 74 | kind string // "s3call", "literal" or "dir" |
| 75 | // enables is how many EnableChangeIndex calls the enclosing function has. |
| 76 | enables int |
| 77 | } |
| 78 | |
| 79 | const releaseImportPath = "github.com/carlosframework/platform/internal/release" |
| 80 | |
| 81 | // scanOpenerSites lists the store constructions in one parsed file. |
| 82 | // inReleasePkg is true for files of package release itself, where the |
| 83 | // constructors and the type are referred to without a qualifier. |
| 84 | func scanOpenerSites(fset *token.FileSet, f *ast.File, rel string, inReleasePkg bool) []openerSite { |
| 85 | qual := "" |
| 86 | for _, im := range f.Imports { |
| 87 | p, _ := strconv.Unquote(im.Path.Value) |
| 88 | if p != releaseImportPath { |
| 89 | continue |
| 90 | } |
| 91 | qual = "release" |
| 92 | if im.Name != nil { |
| 93 | qual = im.Name.Name |
| 94 | } |
| 95 | } |
| 96 | if qual == "" && !inReleasePkg { |
| 97 | return nil |
| 98 | } |
| 99 | // isRelease reports whether e names the identifier in package release. |
| 100 | isRelease := func(e ast.Expr, name string) bool { |
| 101 | switch x := e.(type) { |
| 102 | case *ast.SelectorExpr: |
| 103 | id, ok := x.X.(*ast.Ident) |
| 104 | return ok && qual != "" && id.Name == qual && x.Sel.Name == name |
| 105 | case *ast.Ident: |
| 106 | return inReleasePkg && x.Name == name |
| 107 | } |
| 108 | return false |
| 109 | } |
| 110 | var sites []openerSite |
| 111 | for _, decl := range f.Decls { |
| 112 | fn := "<package>" |
| 113 | var root ast.Node = decl |
| 114 | if fd, ok := decl.(*ast.FuncDecl); ok { |
| 115 | fn = fd.Name.Name |
| 116 | if fd.Body == nil { |
| 117 | continue |
| 118 | } |
| 119 | root = fd.Body |
| 120 | } |
| 121 | enables := 0 |
| 122 | var found []openerSite |
| 123 | ast.Inspect(root, func(n ast.Node) bool { |
| 124 | switch x := n.(type) { |
| 125 | case *ast.CallExpr: |
| 126 | if sel, ok := x.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "EnableChangeIndex" { |
| 127 | enables++ |
| 128 | } |
| 129 | for _, name := range []string{"NewS3Store", "NewS3StoreIn"} { |
| 130 | if isRelease(x.Fun, name) { |
| 131 | found = append(found, openerSite{key: rel + ":" + fn, line: fset.Position(x.Pos()).Line, what: "call to " + name, kind: "s3call"}) |
| 132 | } |
| 133 | } |
| 134 | if isRelease(x.Fun, "NewDirStore") { |
| 135 | found = append(found, openerSite{key: rel + ":" + fn, line: fset.Position(x.Pos()).Line, what: "call to NewDirStore", kind: "dir"}) |
| 136 | } |
| 137 | case *ast.CompositeLit: |
| 138 | if isRelease(x.Type, "S3Store") { |
| 139 | found = append(found, openerSite{key: rel + ":" + fn, line: fset.Position(x.Pos()).Line, what: "S3Store literal", kind: "literal"}) |
| 140 | } |
| 141 | } |
| 142 | return true |
| 143 | }) |
| 144 | for _, s := range found { |
| 145 | s.enables = enables |
| 146 | sites = append(sites, s) |
| 147 | } |
| 148 | } |
| 149 | return sites |
| 150 | } |
| 151 | |
| 152 | func moduleRoot(t *testing.T) string { |
| 153 | t.Helper() |
| 154 | dir, err := os.Getwd() |
| 155 | if err != nil { |
| 156 | t.Fatal(err) |
| 157 | } |
| 158 | for { |
| 159 | if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { |
| 160 | return dir |
| 161 | } |
| 162 | parent := filepath.Dir(dir) |
| 163 | if parent == dir { |
| 164 | t.Fatal("no go.mod above the test directory") |
| 165 | } |
| 166 | dir = parent |
| 167 | } |
| 168 | } |
| 169 | |
| 170 | func TestS3StoreOpenersClassified(t *testing.T) { |
| 171 | root := moduleRoot(t) |
| 172 | fset := token.NewFileSet() |
| 173 | var sites []openerSite |
| 174 | err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { |
| 175 | if err != nil { |
| 176 | return err |
| 177 | } |
| 178 | name := d.Name() |
| 179 | if d.IsDir() { |
| 180 | if path != root && (strings.HasPrefix(name, ".") || name == "vendor" || name == "testdata" || name == "node_modules") { |
| 181 | return filepath.SkipDir |
| 182 | } |
| 183 | if path != root { |
| 184 | if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil { |
| 185 | return filepath.SkipDir // a nested module |
| 186 | } |
| 187 | } |
| 188 | return nil |
| 189 | } |
| 190 | if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { |
| 191 | return nil |
| 192 | } |
| 193 | rel, err := filepath.Rel(root, path) |
| 194 | if err != nil { |
| 195 | return err |
| 196 | } |
| 197 | rel = filepath.ToSlash(rel) |
| 198 | if rel == "internal/release/store.go" { |
| 199 | return nil // the constructors themselves |
| 200 | } |
| 201 | f, err := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution) |
| 202 | if err != nil { |
| 203 | return err |
| 204 | } |
| 205 | sites = append(sites, scanOpenerSites(fset, f, rel, f.Name.Name == "release" && filepath.ToSlash(filepath.Dir(rel)) == "internal/release")...) |
| 206 | return nil |
| 207 | }) |
| 208 | if err != nil { |
| 209 | t.Fatal(err) |
| 210 | } |
| 211 | |
| 212 | byKey := map[string][]openerSite{} |
| 213 | for _, s := range sites { |
| 214 | byKey[s.key] = append(byKey[s.key], s) |
| 215 | } |
| 216 | var bad []string |
| 217 | for key, ss := range byKey { |
| 218 | if e, ok := deploymentManualSites[key]; ok { |
| 219 | for _, s := range ss { |
| 220 | if s.kind == "s3call" { |
| 221 | bad = append(bad, key+" (line "+strconv.Itoa(s.line)+"): "+s.what+" in a deployment site; use NewDeploymentS3Store[In]") |
| 222 | } |
| 223 | } |
| 224 | if len(ss) != e.count { |
| 225 | bad = append(bad, key+": "+strconv.Itoa(len(ss))+" store opens found, deploymentManualSites says "+strconv.Itoa(e.count)+" ("+e.reason+"); classify the new one and update the count") |
| 226 | } |
| 227 | if ss[0].enables < len(ss) { |
| 228 | bad = append(bad, key+": "+strconv.Itoa(len(ss))+" deployment store opens but only "+strconv.Itoa(ss[0].enables)+" EnableChangeIndex calls in the function") |
| 229 | } |
| 230 | continue |
| 231 | } |
| 232 | if e, ok := otherBucketOpeners[key]; ok { |
| 233 | if len(ss) != e.count { |
| 234 | bad = append(bad, key+": "+strconv.Itoa(len(ss))+" store opens found, otherBucketOpeners says "+strconv.Itoa(e.count)+" ("+e.reason+"); classify the new one and update the count") |
| 235 | } |
| 236 | continue |
| 237 | } |
| 238 | for _, s := range ss { |
| 239 | bad = append(bad, key+" (line "+strconv.Itoa(s.line)+"): "+s.what+" is not classified. S3 deployment bucket: use NewDeploymentS3Store[In]. Deployment directory or literal: call EnableChangeIndex and list it in deploymentManualSites. Anything else: add it to otherBucketOpeners with the reason") |
| 240 | } |
| 241 | } |
| 242 | for k := range otherBucketOpeners { |
| 243 | if byKey[k] == nil { |
| 244 | bad = append(bad, k+": listed in otherBucketOpeners but no longer opens a store (remove the entry)") |
| 245 | } |
| 246 | } |
| 247 | for k := range deploymentManualSites { |
| 248 | if byKey[k] == nil { |
| 249 | bad = append(bad, k+": listed in deploymentManualSites but no longer opens a store (remove the entry)") |
| 250 | } |
| 251 | } |
| 252 | sort.Strings(bad) |
| 253 | for _, b := range bad { |
| 254 | t.Error(b) |
| 255 | } |
| 256 | if len(sites) == 0 { |
| 257 | t.Fatal("the scan found no store sites at all; the walk is not reaching the module") |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | // TestOpenerScanRecognisesEveryForm keeps the scanner honest: qualified and |
| 262 | // aliased constructors, literals with and without &, and the enable check. |
| 263 | func TestOpenerScanRecognisesEveryForm(t *testing.T) { |
| 264 | const src = `package x |
| 265 | |
| 266 | import rel "github.com/carlosframework/platform/internal/release" |
| 267 | |
| 268 | func a() { rel.NewS3Store(nil, "b") } |
| 269 | func b() { rel.NewS3StoreIn(nil, "b", "r") } |
| 270 | func c() { _ = &rel.S3Store{Bucket: "b"} } |
| 271 | func d() { s := rel.S3Store{Bucket: "b"}; _ = s } |
| 272 | func e() { s := &rel.S3Store{Bucket: "b"}; s.EnableChangeIndex() } |
| 273 | func f() { rel.NewDeploymentS3Store(nil, "b") } |
| 274 | func g() { rel.NewDirStore("d") } |
| 275 | func h() { d := rel.NewDirStore("d"); d.EnableChangeIndex(); _ = rel.NewDirStore("e") } |
| 276 | ` |
| 277 | fset := token.NewFileSet() |
| 278 | f, err := parser.ParseFile(fset, "x.go", src, 0) |
| 279 | if err != nil { |
| 280 | t.Fatal(err) |
| 281 | } |
| 282 | got := map[string][]openerSite{} |
| 283 | for _, s := range scanOpenerSites(fset, f, "x.go", false) { |
| 284 | got[s.key] = append(got[s.key], s) |
| 285 | } |
| 286 | for _, want := range []string{"x.go:a", "x.go:b", "x.go:c", "x.go:d", "x.go:e", "x.go:g"} { |
| 287 | if len(got[want]) != 1 { |
| 288 | t.Errorf("%s: %d sites, want 1", want, len(got[want])) |
| 289 | } |
| 290 | } |
| 291 | if len(got["x.go:f"]) != 0 { |
| 292 | t.Error("NewDeploymentS3Store is not a plain opener") |
| 293 | } |
| 294 | if len(got["x.go:h"]) != 2 || got["x.go:h"][0].enables != 1 { |
| 295 | t.Errorf("h: want two dir sites and one enable call, got %+v", got["x.go:h"]) |
| 296 | } |
| 297 | if got["x.go:c"][0].kind != "literal" || got["x.go:c"][0].enables != 0 { |
| 298 | t.Errorf("c: literal without enable, got %+v", got["x.go:c"]) |
| 299 | } |
| 300 | if got["x.go:g"][0].kind != "dir" || got["x.go:e"][0].enables != 1 { |
| 301 | t.Errorf("g/e: kinds wrong, got %+v %+v", got["x.go:g"], got["x.go:e"]) |
| 302 | } |
| 303 | } |
| 304 | |
amadan