amadan

carlos / platform Public

release, instancectl: hash a staged build once per file identity

A staged build was hashed whole on every wake, by the agent
(VerifyStaged) and again by root (instance control's staged-exe check):
24 ms for a 9 MB binary on dev, about a quarter of a second for a 100 MB
bundle. release.SHA256Cached keeps each digest under the opened file's
identity: device, inode, size, mtime and ctime to the nanosecond. A write
moves ctime and a reinstall by rename is a new inode, so any change is
hashed again. As git does for its index, only a file unchanged for two
seconds is cached, so a rewrite within one coarse clock tick cannot keep
an identity. Linux only; elsewhere every call hashes, as before. The
identity is read again after the hash, and a file that changed while it
was read is not cached.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 9c268ffccdd80ef7a3023fa044aacae5424c44fd parent f1f8633
6 files changed, +271 −6
  • internal/instancectl/stagedexe.go +7 −5
  • internal/release/digestcache.go +103 −0
  • internal/release/digestcache_linux.go +22 −0
  • internal/release/digestcache_other.go +12 −0
  • internal/release/digestcache_test.go +126 −0
  • internal/release/staged.go +1 −1
diff --git a/internal/instancectl/stagedexe.go b/internal/instancectl/stagedexe.go
index f5753917f..2416e884c 100644
--- a/internal/instancectl/stagedexe.go
+++ b/internal/instancectl/stagedexe.go
@@ -2,10 +2,8 @@ package instancectl
import (
"crypto/sha256"
- "encoding/hex"
"errors"
"fmt"
- "io"
"net/http"
"os"
"path/filepath"
@@ -67,11 +65,15 @@ func (s *Server) validateStagedExe(host string, x StagedExe) error {
return fmt.Errorf("%w: %v", ErrStagedExe, err)
}
defer f.Close()
- h := sha256.New()
- if _, err := io.Copy(h, f); err != nil {
+ // Hashed once per identity of the opened file (release.SHA256Cached):
+ // root alone writes the version store, a write moves the file's ctime
+ // and a reinstall is a new inode, so an unchanged identity is the bytes
+ // already checked.
+ got, err := release.SHA256Cached(f)
+ if err != nil {
return fmt.Errorf("%w: %v", ErrStagedExe, err)
}
- if got := hex.EncodeToString(h.Sum(nil)); got != strings.ToLower(x.Digest) {
+ if got != strings.ToLower(x.Digest) {
return fmt.Errorf("%w: %s has digest %s, the attempt verified %s", ErrStagedExe, want, got, x.Digest)
}
return nil
diff --git a/internal/release/digestcache.go b/internal/release/digestcache.go
new file mode 100644
index 000000000..327cb7b36
--- /dev/null
+++ b/internal/release/digestcache.go
@@ -0,0 +1,103 @@
+package release
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "io"
+ "os"
+ "sync"
+ "time"
+)
+
+// A staged build was hashed whole on every wake, by the agent
+// (VerifyStaged) and again by root (instance control's staged-exe check):
+// 24 ms for a 9 MB binary on a dev box, so a 100 MB bundle paid a quarter
+// of a second a wake. The digest is now kept per file identity: device,
+// inode, size, and the modification and change times to the nanosecond,
+// read from the descriptor being hashed. A write to the file moves its
+// ctime, which no unprivileged caller can set back, and a replacement by
+// rename (how the version store installs a build) is a new inode, so any
+// change is a new identity and is hashed again. Where the platform gives no
+// such identity (fileIdentity is linux-only), every call hashes, as before.
+//
+// File times are coarse (a clock tick), so a file changed moments before
+// it is hashed could be rewritten within the same tick and keep its
+// identity. As git does for its index, a digest is cached only for a file
+// whose last change was at least digestSettle before the hash: any later
+// write then lands at a visibly later time.
+var (
+ digestMu sync.Mutex
+ digestCache = map[string]string{} // identity -> sha256 hex
+ // digestHashHook, when set (tests only), runs each time a file is
+ // actually hashed.
+ digestHashHook func()
+ // digestNow is the clock the settle rule reads (tests move it).
+ digestNow = time.Now
+)
+
+// digestSettle is how long a file must have been unchanged for its digest
+// to be cached.
+const digestSettle = 2 * time.Second
+
+// digestCacheMax bounds the cache: identities of replaced builds are never
+// asked again, so past the bound it starts over rather than grow.
+const digestCacheMax = 4096
+
+func resetDigestCache() {
+ digestMu.Lock()
+ defer digestMu.Unlock()
+ digestCache = map[string]string{}
+}
+
+// SHA256Cached is the sha256 of the open file f, from the cache when f's
+// identity was hashed before. It reads from f's current offset, as
+// io.Copy would, so callers pass a freshly opened file.
+func SHA256Cached(f *os.File) (string, error) {
+ fi, err := f.Stat()
+ if err != nil {
+ return "", err
+ }
+ id, changed, ok := fileIdentity(fi)
+ settled := ok && digestNow().Sub(changed) >= digestSettle
+ if ok {
+ digestMu.Lock()
+ sum, hit := digestCache[id]
+ digestMu.Unlock()
+ if hit {
+ return sum, nil
+ }
+ }
+ if digestHashHook != nil {
+ digestHashHook()
+ }
+ h := sha256.New()
+ if _, err := io.Copy(h, f); err != nil {
+ return "", err
+ }
+ sum := hex.EncodeToString(h.Sum(nil))
+ if settled {
+ // The identity is read again after the hash: a file that changed
+ // while it was read is not cached under either identity.
+ if fi2, err := f.Stat(); err == nil {
+ if id2, _, ok2 := fileIdentity(fi2); ok2 && id2 == id {
+ digestMu.Lock()
+ if len(digestCache) >= digestCacheMax {
+ digestCache = map[string]string{}
+ }
+ digestCache[id] = sum
+ digestMu.Unlock()
+ }
+ }
+ }
+ return sum, nil
+}
+
+// FileSHA256Cached is SHA256Cached on the file at path.
+func FileSHA256Cached(path string) (string, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return "", err
+ }
+ defer f.Close()
+ return SHA256Cached(f)
+}
diff --git a/internal/release/digestcache_linux.go b/internal/release/digestcache_linux.go
new file mode 100644
index 000000000..d3bbb8362
--- /dev/null
+++ b/internal/release/digestcache_linux.go
@@ -0,0 +1,22 @@
+package release
+
+import (
+ "fmt"
+ "os"
+ "syscall"
+ "time"
+)
+
+// fileIdentity is f's device, inode, size, mtime and ctime (nanoseconds),
+// and the later of its mtime and ctime: when it last changed.
+func fileIdentity(fi os.FileInfo) (string, time.Time, bool) {
+ st, ok := fi.Sys().(*syscall.Stat_t)
+ if !ok || !fi.Mode().IsRegular() {
+ return "", time.Time{}, false
+ }
+ m, c := time.Unix(st.Mtim.Sec, st.Mtim.Nsec), time.Unix(st.Ctim.Sec, st.Ctim.Nsec)
+ if m.After(c) {
+ c = m
+ }
+ return fmt.Sprintf("%d:%d:%d:%d.%d:%d.%d", st.Dev, st.Ino, st.Size, st.Mtim.Sec, st.Mtim.Nsec, st.Ctim.Sec, st.Ctim.Nsec), c, true
+}
diff --git a/internal/release/digestcache_other.go b/internal/release/digestcache_other.go
new file mode 100644
index 000000000..4daca3830
--- /dev/null
+++ b/internal/release/digestcache_other.go
@@ -0,0 +1,12 @@
+//go:build !linux
+
+package release
+
+import (
+ "os"
+ "time"
+)
+
+// fileIdentity has no portable inode and change time here: every call
+// hashes (digestcache.go).
+func fileIdentity(os.FileInfo) (string, time.Time, bool) { return "", time.Time{}, false }
diff --git a/internal/release/digestcache_test.go b/internal/release/digestcache_test.go
new file mode 100644
index 000000000..58742b448
--- /dev/null
+++ b/internal/release/digestcache_test.go
@@ -0,0 +1,126 @@
+//go:build linux
+
+package release
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+func sumOf(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
+
+// settledClock moves the cache's clock a minute ahead, so the files a test
+// just wrote count as settled.
+func settledClock(t *testing.T) {
+ t.Helper()
+ digestNow = func() time.Time { return time.Now().Add(time.Minute) }
+ t.Cleanup(func() { digestNow = time.Now })
+}
+
+// A file changed moments before it is hashed is not cached: a rewrite in
+// the same clock tick could keep its identity.
+func TestFileSHA256CachedDoesNotCacheAFreshFile(t *testing.T) {
+ resetDigestCache()
+ path := filepath.Join(t.TempDir(), "app")
+ if err := os.WriteFile(path, []byte("build one"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ hashed := 0
+ digestHashHook = func() { hashed++ }
+ t.Cleanup(func() { digestHashHook = nil })
+ for range 2 {
+ if _, err := FileSHA256Cached(path); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if hashed != 2 {
+ t.Fatalf("hashed %d times, want every time for a file not yet settled", hashed)
+ }
+}
+
+// A staged build is hashed once per file identity, not once per wake: the
+// second check of an unchanged file reads none of it.
+func TestFileSHA256CachedHashesAnUnchangedFileOnce(t *testing.T) {
+ resetDigestCache()
+ settledClock(t)
+ path := filepath.Join(t.TempDir(), "app")
+ if err := os.WriteFile(path, []byte("build one"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ hashed := 0
+ digestHashHook = func() { hashed++ }
+ t.Cleanup(func() { digestHashHook = nil })
+ for range 3 {
+ got, err := FileSHA256Cached(path)
+ if err != nil || got != sumOf([]byte("build one")) {
+ t.Fatalf("FileSHA256Cached = %q, %v", got, err)
+ }
+ }
+ if hashed != 1 {
+ t.Fatalf("hashed %d times, want once", hashed)
+ }
+}
+
+// Any change to the file is a new identity: rewritten in place (its mtime
+// and ctime move, whatever mtime is set back to) or replaced by a rename (a
+// new inode). Each is hashed again, so a changed copy is always caught.
+func TestFileSHA256CachedSeesEveryChange(t *testing.T) {
+ resetDigestCache()
+ settledClock(t)
+ dir := t.TempDir()
+ path := filepath.Join(dir, "app")
+ if err := os.WriteFile(path, []byte("build one"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := FileSHA256Cached(path); err != nil {
+ t.Fatal(err)
+ }
+ // In place, same size, mtime put back: ctime still moves.
+ st, _ := os.Stat(path)
+ time.Sleep(10 * time.Millisecond)
+ if err := os.WriteFile(path, []byte("build two"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chtimes(path, st.ModTime(), st.ModTime()); err != nil {
+ t.Fatal(err)
+ }
+ if got, err := FileSHA256Cached(path); err != nil || got != sumOf([]byte("build two")) {
+ t.Fatalf("after an in-place rewrite = %q, %v", got, err)
+ }
+ // Replaced by rename.
+ tmp := filepath.Join(dir, "app.new")
+ if err := os.WriteFile(tmp, []byte("build three"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Rename(tmp, path); err != nil {
+ t.Fatal(err)
+ }
+ if got, err := FileSHA256Cached(path); err != nil || got != sumOf([]byte("build three")) {
+ t.Fatalf("after a rename = %q, %v", got, err)
+ }
+}
+
+// VerifyStaged goes through the cache and still refuses a changed copy.
+func TestVerifyStagedStillRefusesAChangedCopy(t *testing.T) {
+ resetDigestCache()
+ settledClock(t)
+ path := filepath.Join(t.TempDir(), "app")
+ if err := os.WriteFile(path, []byte("build one"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ id := StagedIdentity{Version: "v1", StagedPath: path, Digest: sumOf([]byte("build one"))}
+ if err := VerifyStaged(id); err != nil {
+ t.Fatal(err)
+ }
+ time.Sleep(10 * time.Millisecond)
+ if err := os.WriteFile(path, []byte("tampered!"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := VerifyStaged(id); err == nil {
+ t.Fatal("a changed copy verified")
+ }
+}
diff --git a/internal/release/staged.go b/internal/release/staged.go
index 8487a0743..df1e43e28 100644
--- a/internal/release/staged.go
+++ b/internal/release/staged.go
@@ -180,7 +180,7 @@ func VerifyStaged(id StagedIdentity) error {
if id.StagedPath == "" || id.Digest == "" {
return fmt.Errorf("%w: no retained path and digest for %q", ErrStagedCopyChanged, id.Version)
}
- sum, err := FileSHA256(id.StagedPath)
+ sum, err := FileSHA256Cached(id.StagedPath)
if err != nil {
return fmt.Errorf("%w: %s: %v", ErrStagedCopyChanged, id.StagedPath, err)
}