amadan

carlos / platform Public

host: the mint asker re-hashes a staged file that changed, and caches proofs per proof object (H5)

shaByPath assumed version directories are immutable, but a pruned version name can be reused for other bytes at the same path; the asker then answered from the old hash and its proof and never asked root to mint the new bytes. The hash cache now holds the file's size, mtime and (on unix) device and inode, and re-hashes when any differ. The proven cache is keyed on (account, app, sha256, prover) instead of the host.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 303cc13911bc3b034701a818e6b71618af719750 parent 58dd5ba
4 files changed, +171 −12
  • internal/host/fileowner_other.go +4 −0
  • internal/host/fileowner_unix.go +10 −0
  • internal/host/proofask.go +60 −12
  • internal/host/proofask_test.go +97 −0
diff --git a/internal/host/fileowner_other.go b/internal/host/fileowner_other.go
index 9ab9347f4..cb573760a 100644
--- a/internal/host/fileowner_other.go
+++ b/internal/host/fileowner_other.go
@@ -8,3 +8,7 @@ import "io/fs"
// about ownership" rather than as a failure, so the CLI still builds and runs
// on Windows; the box side this serves is Linux-only anyway.
func fileOwner(fs.FileInfo) (uid, gid int, ok bool) { return 0, 0, false }
+
+// fileInode has no answer off unix either; a caller keys on size and mtime
+// alone there.
+func fileInode(fs.FileInfo) (dev, ino uint64, ok bool) { return 0, 0, false }
diff --git a/internal/host/fileowner_unix.go b/internal/host/fileowner_unix.go
index 1a8067ba6..d88fe790b 100644
--- a/internal/host/fileowner_unix.go
+++ b/internal/host/fileowner_unix.go
@@ -18,3 +18,13 @@ func fileOwner(fi fs.FileInfo) (uid, gid int, ok bool) {
}
return int(st.Uid), int(st.Gid), true
}
+
+// fileInode reports a file's device and inode, so a cache can tell a file
+// replaced at the same path from the one it hashed. Same split as fileOwner.
+func fileInode(fi fs.FileInfo) (dev, ino uint64, ok bool) {
+ st, ok := fi.Sys().(*syscall.Stat_t)
+ if !ok {
+ return 0, 0, false
+ }
+ return uint64(st.Dev), uint64(st.Ino), true
+}
diff --git a/internal/host/proofask.go b/internal/host/proofask.go
index 0c2648aca..d3cce2db6 100644
--- a/internal/host/proofask.go
+++ b/internal/host/proofask.go
@@ -57,9 +57,59 @@ type mintAsker struct {
now func() time.Time
last map[string]time.Time // host -> when it was last looked at
- proven map[string]string // host -> sha256 seen proven
+ proven map[provenKey]bool // proofs seen in the bucket
pending map[string]string // host -> sha256 whose pending note is written
- shaByPath map[string]string // staged file -> sha256 (version dirs are immutable)
+ shaByPath map[string]hashedFile
+}
+
+// hashedFile is a staged file's sha256 and the identity of the file it was
+// taken from. A version directory can be pruned and its version name reused
+// for other bytes at the same path, so a cached hash is used only while the
+// file is still the same one: same size, mtime and, where the platform says,
+// device and inode (final review H5).
+type hashedFile struct {
+ size int64
+ mtime int64
+ dev, ino uint64
+ sha string
+}
+
+func identify(fi os.FileInfo) hashedFile {
+ dev, ino, _ := fileInode(fi)
+ return hashedFile{size: fi.Size(), mtime: fi.ModTime().UnixNano(), dev: dev, ino: ino}
+}
+
+func (h hashedFile) sameFile(o hashedFile) bool {
+ return h.size == o.size && h.mtime == o.mtime && h.dev == o.dev && h.ino == o.ino
+}
+
+// provenKey is one proof object: a sha256 the bucket holds a proof for,
+// under one account, app and prover. A host is not part of it, and neither
+// a change of prover nor of account can reuse another's answer.
+type provenKey struct{ acct, app, sha, prover string }
+
+// stagedSHA is target's sha256, hashed again whenever the file is not the
+// one the cached hash came from.
+func (a *mintAsker) stagedSHA(target string) (string, error) {
+ fi, err := os.Stat(target)
+ if err != nil {
+ return "", err
+ }
+ id := identify(fi)
+ if h, ok := a.shaByPath[target]; ok && h.sameFile(id) {
+ return h.sha, nil
+ }
+ sha, err := release.FileSHA256(target)
+ if err != nil {
+ return "", err
+ }
+ // The file can change between the stat and the read; a hash taken across
+ // that change is used once and not cached.
+ if fi2, err := os.Stat(target); err == nil && identify(fi2).sameFile(id) {
+ id.sha = sha
+ a.shaByPath[target] = id
+ }
+ return sha, nil
}
func (a *mintAsker) tick(ctx context.Context) {
@@ -76,7 +126,7 @@ func (a *mintAsker) tick(ctx context.Context) {
return
}
if a.last == nil {
- a.last, a.proven, a.pending, a.shaByPath = map[string]time.Time{}, map[string]string{}, map[string]string{}, map[string]string{}
+ a.last, a.proven, a.pending, a.shaByPath = map[string]time.Time{}, map[provenKey]bool{}, map[string]string{}, map[string]hashedFile{}
}
now := a.now()
after := pol.ProveAfter()
@@ -93,18 +143,16 @@ func (a *mintAsker) tick(ctx context.Context) {
if err != nil {
continue
}
- sha, ok := a.shaByPath[target]
- if !ok {
- if sha, err = release.FileSHA256(target); err != nil {
- continue
- }
- a.shaByPath[target] = sha
+ sha, err := a.stagedSHA(target)
+ if err != nil {
+ continue
}
- if a.proven[rt.Host] == sha {
+ acct := release.RouteAccount(rt)
+ pk := provenKey{acct, rt.App, sha, prover}
+ if a.proven[pk] {
continue
}
ver, _ := release.LiveVersion(a.optDir, rt.Host)
- acct := release.RouteAccount(rt)
if p, err := a.sys.Show(rt.Unit); err == nil {
if p.ActiveState != "active" {
continue
@@ -127,7 +175,7 @@ func (a *mintAsker) tick(ctx context.Context) {
_, err = release.GetProof(ctx, a.store, acct, rt.App, sha, prover)
switch {
case err == nil:
- a.proven[rt.Host] = sha
+ a.proven[pk] = true
continue
case !errors.Is(err, os.ErrNotExist):
log.Printf("agent: %s: reading the bucket for %s's proof: %v", rt.Host, ver, err)
diff --git a/internal/host/proofask_test.go b/internal/host/proofask_test.go
index 284d29862..b64fbf476 100644
--- a/internal/host/proofask_test.go
+++ b/internal/host/proofask_test.go
@@ -89,6 +89,103 @@ func TestMintAskerReasksUntilTheProofAppears(t *testing.T) {
}
}
+// putDevProof puts a valid-shaped dev proof for sha in store.
+func putDevProof(t *testing.T, store release.Store, sha string, at time.Time) {
+ t.Helper()
+ seed, _, err := release.GenerateKey()
+ if err != nil {
+ t.Fatal(err)
+ }
+ key, err := release.ParseSigningKey(seed, "test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ p, err := release.SignProof(key, release.Proof{App: "console", Platform: release.AdoptPlatform(), SHA256: sha,
+ Version: "v1", Prover: "dev", ActiveSince: at.Add(-20 * time.Minute), ProvenAt: at})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := release.PutProof(context.Background(), store, "bab", p); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// Final review H5: dev proves v1, its staged copy is pruned, and the name v1
+// is reused for other bytes at the same path. The asker hashes the new file,
+// finds no proof for it, and asks root again, rather than answering from the
+// old file's hash and the proof it found for that.
+func TestMintAskerRehashesAReusedVersionPath(t *testing.T) {
+ a, asked, clock, store, sha := mintAskerFixture(t, 20*time.Minute)
+ ctx := context.Background()
+ t0 := *clock
+ putDevProof(t, store, sha, t0)
+ a.tick(ctx)
+ if *asked != 0 {
+ t.Fatalf("asked %d times for a proven build, want 0", *asked)
+ }
+ staged := filepath.Join(a.optDir, "versions", "v1", "console")
+ if err := os.Remove(staged); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(staged, []byte("other bytes under the same name"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ *clock = t0.Add(mintAskEvery)
+ a.tick(ctx)
+ if *asked != 1 {
+ t.Fatalf("asked %d times after the path's bytes changed, want 1: the new bytes have no proof", *asked)
+ }
+ newSHA := sha256Hex("other bytes under the same name")
+ if h := a.shaByPath[staged]; h.sha != newSHA {
+ t.Fatalf("cached sha256 = %q, want the new file's %q", h.sha, newSHA)
+ }
+}
+
+// The same bytes rewritten in place with a new mtime are hashed again too:
+// size and mtime are part of the file's identity, not only its inode.
+func TestMintAskerRehashesAFileRewrittenInPlace(t *testing.T) {
+ a, asked, clock, store, sha := mintAskerFixture(t, 20*time.Minute)
+ ctx := context.Background()
+ t0 := *clock
+ putDevProof(t, store, sha, t0)
+ a.tick(ctx)
+ staged := filepath.Join(a.optDir, "versions", "v1", "console")
+ // liveTo's build is one byte; another byte of the same size, in place.
+ if err := os.WriteFile(staged, []byte("y"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ later := time.Now().Add(time.Hour)
+ if err := os.Chtimes(staged, later, later); err != nil {
+ t.Fatal(err)
+ }
+ *clock = t0.Add(mintAskEvery)
+ a.tick(ctx)
+ if *asked != 1 {
+ t.Fatalf("asked %d times after an in-place rewrite, want 1", *asked)
+ }
+}
+
+// The proven cache is per proof object: a proof seen for one prover does not
+// answer for another after the policy changes prover.
+func TestMintAskerProvenCacheIsPerProver(t *testing.T) {
+ a, asked, clock, store, sha := mintAskerFixture(t, 20*time.Minute)
+ ctx := context.Background()
+ t0 := *clock
+ putDevProof(t, store, sha, t0)
+ a.tick(ctx)
+ if *asked != 0 {
+ t.Fatalf("asked %d times for a proven build, want 0", *asked)
+ }
+ if err := os.WriteFile(a.policyPath, []byte(`{"schema":1,"prove":{"apps":["console"],"prover":"staging","after":"15m","key_file":"/etc/carlos/proving.key"}}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ *clock = t0.Add(mintAskEvery)
+ a.tick(ctx)
+ if *asked != 1 {
+ t.Fatalf("asked %d times once the prover is staging, want 1: dev's proof is not staging's", *asked)
+ }
+}
+
func TestMintAskerWaitsOutTheBake(t *testing.T) {
a, asked, _, store, sha := mintAskerFixture(t, 5*time.Minute)
a.tick(context.Background())