amadan
hostfiles: manifest entries name their objects by origin-qualified key, restore reads only admitted origins, and CopyManifest and Publish are the exported contracts
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
10 files changed,
+613
−86
internal/hostfiles/contract.go+112 −0internal/hostfiles/hostfiles_test.go+43 −43internal/hostfiles/manifest.go+55 −3internal/hostfiles/origins_test.go+304 −0internal/hostfiles/publisher.go+7 −6internal/hostfiles/restore.go+63 −25internal/hostfiles/sync_test.go+2 −2internal/hostfiles/sync_unix_test.go+20 −0internal/hostfiles/wake.go+3 −3internal/hostfiles/wake_test.go+4 −4
diff --git a/internal/hostfiles/contract.go b/internal/hostfiles/contract.go| new file mode 100644 |
| index 000000000..7f142e08d |
| --- /dev/null |
| +++ b/internal/hostfiles/contract.go |
| @@ -0,0 +1,112 @@ |
| +package hostfiles |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "os" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// This file holds the entry points other packages call: region HA's claim |
| +// (CopyManifest, at a new generation's admission) and moves (Publish, at |
| +// install). Both are dormant: nothing in a production path calls them yet. |
| + |
| +// ErrManifestExists means CopyManifest's target already holds a different |
| +// manifest. The copy wrote nothing. |
| +var ErrManifestExists = errors.New("hostfiles: the target generation already has a manifest") |
| + |
| +// CopyManifest establishes toPrefix's manifest as an exact copy of |
| +// fromPrefix's: the same sequence number and the same entries, each still |
| +// naming its object by its origin-qualified key. No object is copied; the |
| +// new generation's publisher continues the sequence from the copy and |
| +// writes new objects under toPrefix only. |
| +// |
| +// origins are the prefixes the source's entries may reference, as for |
| +// Restore (nil or empty: fromPrefix alone). A source built by earlier |
| +// copies references earlier generations, so its caller passes the chain |
| +// its pin admits. Every entry is validated before anything is written; an |
| +// absent source is os.ErrNotExist, never an empty copy. |
| +// |
| +// The write is create-only. A target that already holds exactly these |
| +// bytes (a repeat whose first response was lost) is success; any other |
| +// target is ErrManifestExists and is left as it was. |
| +func CopyManifest(ctx context.Context, s release.Store, fromPrefix, toPrefix string, origins []string) (Manifest, error) { |
| + if err := checkPrefix(fromPrefix); err != nil { |
| + return Manifest{}, err |
| + } |
| + if err := checkPrefix(toPrefix); err != nil { |
| + return Manifest{}, err |
| + } |
| + if fromPrefix == toPrefix { |
| + return Manifest{}, fmt.Errorf("hostfiles: copy %s onto itself", fromPrefix) |
| + } |
| + origins, err := admitted(fromPrefix, origins) |
| + if err != nil { |
| + return Manifest{}, err |
| + } |
| + m, _, found, err := load(ctx, s, fromPrefix) |
| + if err != nil { |
| + return Manifest{}, err |
| + } |
| + if !found { |
| + return Manifest{}, fmt.Errorf("hostfiles: copy from %s: %w", fromPrefix, os.ErrNotExist) |
| + } |
| + if err := checkManifest(m, origins); err != nil { |
| + return Manifest{}, err |
| + } |
| + body, err := json.Marshal(m) |
| + if err != nil { |
| + return Manifest{}, fmt.Errorf("hostfiles: marshal manifest: %w", err) |
| + } |
| + if _, err := s.Put(ctx, ManifestKey(toPrefix), body, ""); errors.Is(err, release.ErrCASConflict) { |
| + cur, _, gerr := s.Get(ctx, ManifestKey(toPrefix)) |
| + if gerr != nil { |
| + return Manifest{}, gerr |
| + } |
| + if !bytes.Equal(cur, body) { |
| + return Manifest{}, fmt.Errorf("%w: %s", ErrManifestExists, toPrefix) |
| + } |
| + } else if err != nil { |
| + return Manifest{}, err |
| + } |
| + return m, nil |
| +} |
| + |
| +// Publish records dir as prefix's manifest in one revision and returns |
| +// that revision's sequence number: one SyncDir and one Commit on a fresh |
| +// Publisher. It is the call moves makes at install, after the directory is |
| +// in place and before the installation is recorded. |
| +// |
| +// skip is SyncDir's: it excludes a relative path and, for a directory, |
| +// everything under it. On success every regular file in dir that skip does |
| +// not exclude is in the manifest: published, or recorded as skipped when |
| +// it is larger than MaxFileBytes (st.TooLarge). Anything that is not a |
| +// regular file is left out and listed in st.NotRegular. A file that |
| +// exists but cannot be read is an error, and nothing is committed: the |
| +// manifest would otherwise not hold it. |
| +// |
| +// A prefix that already has a manifest is continued, not replaced: a |
| +// repeat with nothing changed returns the same sequence number. Another |
| +// writer's manifest appearing between Open and Commit is ErrSuperseded. |
| +// Platform items (wake state, an env file) are not part of this call; a |
| +// caller that needs them in the same revision uses Open, StagePlatform, |
| +// SyncDir and Commit itself. |
| +func Publish(ctx context.Context, s release.Store, prefix, dir string, skip func(rel string, isDir bool) bool) (int64, SyncStats, error) { |
| + p, err := Open(ctx, s, prefix) |
| + if err != nil { |
| + return 0, SyncStats{}, err |
| + } |
| + st, err := SyncDir(ctx, p, dir, skip) |
| + if err != nil { |
| + return 0, st, err |
| + } |
| + if len(st.Unreadable) > 0 { |
| + return 0, st, fmt.Errorf("hostfiles: publish %s: %d file(s) could not be read: %q", dir, len(st.Unreadable), st.Unreadable) |
| + } |
| + seq, err := p.Commit(ctx) |
| + return seq, st, err |
| +} |
diff --git a/internal/hostfiles/hostfiles_test.go b/internal/hostfiles/hostfiles_test.go| index ca1135232..366d53543 100644 |
| --- a/internal/hostfiles/hostfiles_test.go |
| +++ b/internal/hostfiles/hostfiles_test.go |
| @@ -58,7 +58,7 @@ func TestPublishThenRestoreRoundTrips(t *testing.T) { |
| t.Fatalf("seq = %d, want 2", seq) |
| } |
| dir := t.TempDir() |
| - m, st, err := Restore(ctx, s, pfx, dir) |
| + m, st, err := Restore(ctx, s, pfx, dir, nil) |
| if err != nil { |
| t.Fatal(err) |
| } |
| @@ -85,7 +85,7 @@ func TestALaterWriteWinsAndAnUnchangedOneTakesNoSequence(t *testing.T) { |
| t.Fatalf("seq = %d, want 2", seq) |
| } |
| dir := t.TempDir() |
| - if _, _, err := Restore(ctx, s, pfx, dir); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err != nil { |
| t.Fatal(err) |
| } |
| if got := read(t, dir, "instance.key"); got != "new" { |
| @@ -105,7 +105,7 @@ func TestALateUploadCannotResurrectOldContent(t *testing.T) { |
| t.Fatal(err) |
| } |
| dir := t.TempDir() |
| - if _, _, err := Restore(ctx, s, pfx, dir); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err != nil { |
| t.Fatal(err) |
| } |
| if got := read(t, dir, "instance.key"); got != "new" { |
| @@ -120,14 +120,14 @@ func TestADeleteIsATombstoneAndRestoreRemovesTheFile(t *testing.T) { |
| stage(t, p, "b.txt", "b") |
| commit(t, p) |
| dir := t.TempDir() |
| - if _, _, err := Restore(ctx, s, pfx, dir); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err != nil { |
| t.Fatal(err) |
| } |
| if err := p.StageDelete("a.txt"); err != nil { |
| t.Fatal(err) |
| } |
| commit(t, p) |
| - _, st, err := Restore(ctx, s, pfx, dir) |
| + _, st, err := Restore(ctx, s, pfx, dir, nil) |
| if err != nil { |
| t.Fatal(err) |
| } |
| @@ -152,7 +152,7 @@ func TestASecondWriterIsSupersededNotMerged(t *testing.T) { |
| t.Fatalf("err = %v, want ErrSuperseded", err) |
| } |
| dir := t.TempDir() |
| - if _, _, err := Restore(context.Background(), s, pfx, dir); err != nil { |
| + if _, _, err := Restore(context.Background(), s, pfx, dir, nil); err != nil { |
| t.Fatal(err) |
| } |
| if got := read(t, dir, "x"); got != "from a" { |
| @@ -193,7 +193,7 @@ func TestStageAndRestoreRefuseAnEscapingPath(t *testing.T) { |
| t.Fatal("Stage accepted ../evil") |
| } |
| // A manifest written by something else with a hostile path. |
| - body := `{"v":1,"seq":1,"entries":{"../evil":{"object":"objects/0000000000000001-aa","sha256":"aa","seq":1}},"platform":{}}` |
| + body := `{"v":2,"seq":1,"entries":{"../evil":{"object":"objects/0000000000000001-aa","sha256":"aa","seq":1}},"platform":{}}` |
| hostile := "instances/h/i/files/g/" |
| if _, err := s.Put(ctx, ManifestKey(hostile), []byte(body), ""); err != nil { |
| t.Fatal(err) |
| @@ -203,7 +203,7 @@ func TestStageAndRestoreRefuseAnEscapingPath(t *testing.T) { |
| if err := os.Mkdir(dir, 0o755); err != nil { |
| t.Fatal(err) |
| } |
| - if _, _, err := Restore(ctx, s, hostile, dir); err == nil { |
| + if _, _, err := Restore(ctx, s, hostile, dir, nil); err == nil { |
| t.Fatal("Restore accepted a manifest with ../evil") |
| } |
| if _, err := os.Stat(filepath.Join(parent, "evil")); !os.IsNotExist(err) { |
| @@ -220,18 +220,18 @@ func TestRestoreRefusesContentThatDoesNotMatchItsHash(t *testing.T) { |
| if err != nil { |
| t.Fatal(err) |
| } |
| - key := pfx + m.Entries["x"].Object |
| + key := m.Entries["x"].Object |
| _, etag, _ := s.Get(ctx, key) |
| if _, err := s.Put(ctx, key, []byte("tampered"), etag); err != nil { |
| t.Fatal(err) |
| } |
| - if _, _, err := Restore(ctx, s, pfx, t.TempDir()); err == nil || !strings.Contains(err.Error(), "sha256") { |
| + if _, _, err := Restore(ctx, s, pfx, t.TempDir(), nil); err == nil || !strings.Contains(err.Error(), "sha256") { |
| t.Fatalf("err = %v, want a sha256 mismatch", err) |
| } |
| } |
| func TestAnAbsentManifestRestoresNothing(t *testing.T) { |
| - m, st, err := Restore(context.Background(), release.NewDirStore(t.TempDir()), pfx, t.TempDir()) |
| + m, st, err := Restore(context.Background(), release.NewDirStore(t.TempDir()), pfx, t.TempDir(), nil) |
| if err != nil || m.Seq != 0 || st.Written != 0 { |
| t.Fatalf("m=%+v st=%+v err=%v", m, st, err) |
| } |
| @@ -245,18 +245,18 @@ func TestPlatformItemsAreNotWrittenIntoTheFilesDirectory(t *testing.T) { |
| } |
| commit(t, p) |
| dir := t.TempDir() |
| - m, _, err := Restore(ctx, s, pfx, dir) |
| + m, _, err := Restore(ctx, s, pfx, dir, nil) |
| if err != nil { |
| t.Fatal(err) |
| } |
| if entries, _ := os.ReadDir(dir); len(entries) != 0 { |
| t.Fatalf("files directory has %d entries, want 0", len(entries)) |
| } |
| - got, err := ReadPlatform(ctx, s, pfx, m, "wake.json") |
| + got, err := ReadPlatform(ctx, s, pfx, m, "wake.json", nil) |
| if err != nil || string(got) != `{"v":1}` { |
| t.Fatalf("ReadPlatform = %q, %v", got, err) |
| } |
| - if _, err := ReadPlatform(ctx, s, pfx, m, "absent"); !errors.Is(err, os.ErrNotExist) { |
| + if _, err := ReadPlatform(ctx, s, pfx, m, "absent", nil); !errors.Is(err, os.ErrNotExist) { |
| t.Fatalf("absent item: err = %v, want os.ErrNotExist", err) |
| } |
| } |
| @@ -269,7 +269,7 @@ func TestASkippedFileIsRecordedAndReportedOnRestore(t *testing.T) { |
| t.Fatal(err) |
| } |
| commit(t, p) |
| - _, st, err := Restore(ctx, s, pfx, t.TempDir()) |
| + _, st, err := Restore(ctx, s, pfx, t.TempDir(), nil) |
| if err != nil { |
| t.Fatal(err) |
| } |
| @@ -300,7 +300,7 @@ func TestRestoreNeverWritesOrDeletesThroughASymlink(t *testing.T) { |
| t.Fatal(err) |
| } |
| } |
| - if _, _, err := Restore(ctx, s, pfx, dir); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err != nil { |
| t.Fatal(err) |
| } |
| if _, err := os.Stat(filepath.Join(outside, "x")); !os.IsNotExist(err) { |
| @@ -323,7 +323,7 @@ func TestRestoreHandlesADirectoryBecomingAFileAndBack(t *testing.T) { |
| stage(t, p, "a/b", "child") |
| commit(t, p) |
| prior := t.TempDir() // a copy restored while "a" was a directory |
| - if _, _, err := Restore(ctx, s, pfx, prior); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, prior, nil); err != nil { |
| t.Fatal(err) |
| } |
| @@ -334,7 +334,7 @@ func TestRestoreHandlesADirectoryBecomingAFileAndBack(t *testing.T) { |
| stage(t, p, "a", "now a file") |
| commit(t, p) |
| for name, dir := range map[string]string{"an empty destination": t.TempDir(), "a previously restored one": prior} { |
| - if _, _, err := Restore(ctx, s, pfx, dir); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err != nil { |
| t.Fatalf("%s: %v", name, err) |
| } |
| if got := read(t, dir, "a"); got != "now a file" { |
| @@ -349,7 +349,7 @@ func TestRestoreHandlesADirectoryBecomingAFileAndBack(t *testing.T) { |
| stage(t, p, "a/c", "child again") |
| commit(t, p) |
| for name, dir := range map[string]string{"an empty destination": t.TempDir(), "a previously restored one": prior} { |
| - if _, _, err := Restore(ctx, s, pfx, dir); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err != nil { |
| t.Fatalf("%s: %v", name, err) |
| } |
| if got := read(t, dir, "a/c"); got != "child again" { |
| @@ -363,7 +363,7 @@ func TestRestoreIntoAMissingDirectoryIsAnError(t *testing.T) { |
| p := open(t, s) |
| stage(t, p, "x", "x") |
| commit(t, p) |
| - if _, _, err := Restore(ctx, s, pfx, filepath.Join(t.TempDir(), "absent")); err == nil { |
| + if _, _, err := Restore(ctx, s, pfx, filepath.Join(t.TempDir(), "absent"), nil); err == nil { |
| t.Fatal("restored into a directory that does not exist") |
| } |
| } |
| @@ -378,7 +378,7 @@ func TestRestoreSyncsEveryDirectoryItChangedBeforeReturning(t *testing.T) { |
| stage(t, p, "gone/old.txt", "o") |
| commit(t, p) |
| dir := t.TempDir() |
| - if _, _, err := Restore(ctx, s, pfx, dir); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err != nil { |
| t.Fatal(err) |
| } |
| if err := p.StageDelete("gone/old.txt"); err != nil { |
| @@ -399,7 +399,7 @@ func TestRestoreSyncsEveryDirectoryItChangedBeforeReturning(t *testing.T) { |
| synced = append(synced, rel) |
| return orig(root, rel) |
| } |
| - if _, _, err := Restore(ctx, s, pfx, dir); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err != nil { |
| t.Fatal(err) |
| } |
| want := []string{"a/b", "a", "gone", "."} |
| @@ -408,7 +408,7 @@ func TestRestoreSyncsEveryDirectoryItChangedBeforeReturning(t *testing.T) { |
| } |
| syncDir = func(*os.Root, string) error { return errors.New("disk says no") } |
| - if _, _, err := Restore(ctx, s, pfx, dir); err == nil || !strings.Contains(err.Error(), "disk says no") { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err == nil || !strings.Contains(err.Error(), "disk says no") { |
| t.Fatalf("err = %v; a failed directory sync must fail the restore", err) |
| } |
| } |
| @@ -422,7 +422,7 @@ func TestARetriedRestoreStillSyncsADeletionAnEarlierAttemptMade(t *testing.T) { |
| stage(t, p, "gone/old.txt", "o") |
| commit(t, p) |
| dir := t.TempDir() |
| - if _, _, err := Restore(ctx, s, pfx, dir); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err != nil { |
| t.Fatal(err) |
| } |
| if err := p.StageDelete("gone/old.txt"); err != nil { |
| @@ -433,7 +433,7 @@ func TestARetriedRestoreStillSyncsADeletionAnEarlierAttemptMade(t *testing.T) { |
| orig := syncDir |
| t.Cleanup(func() { syncDir = orig }) |
| syncDir = func(*os.Root, string) error { return errors.New("disk says no") } |
| - if _, _, err := Restore(ctx, s, pfx, dir); err == nil { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err == nil { |
| t.Fatal("the first attempt must fail") |
| } |
| if _, err := os.Stat(filepath.Join(dir, "gone", "old.txt")); !os.IsNotExist(err) { |
| @@ -441,7 +441,7 @@ func TestARetriedRestoreStillSyncsADeletionAnEarlierAttemptMade(t *testing.T) { |
| } |
| var synced []string |
| syncDir = func(root *os.Root, rel string) error { synced = append(synced, rel); return orig(root, rel) } |
| - _, st, err := Restore(ctx, s, pfx, dir) |
| + _, st, err := Restore(ctx, s, pfx, dir, nil) |
| if err != nil { |
| t.Fatal(err) |
| } |
| @@ -462,7 +462,7 @@ func TestRestoreHandlesADeepDirectoryBecomingAFile(t *testing.T) { |
| stage(t, p, "a/b/c", "leaf") |
| commit(t, p) |
| dir := t.TempDir() |
| - if _, _, err := Restore(ctx, s, pfx, dir); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err != nil { |
| t.Fatal(err) |
| } |
| if err := p.StageDelete("a/b/c"); err != nil { |
| @@ -477,7 +477,7 @@ func TestRestoreHandlesADeepDirectoryBecomingAFile(t *testing.T) { |
| syncDir = func(root *os.Root, rel string) error { synced = append(synced, rel); return orig(root, rel) } |
| for name, d := range map[string]string{"a previously restored destination": dir, "an empty one": t.TempDir()} { |
| synced = nil |
| - if _, _, err := Restore(ctx, s, pfx, d); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, d, nil); err != nil { |
| t.Fatalf("%s: %v", name, err) |
| } |
| if got := read(t, d, "a"); got != "now a file" { |
| @@ -554,7 +554,7 @@ func TestRestoreMasksTheModeToPermissionBits(t *testing.T) { |
| t.Fatal(err) |
| } |
| dir := t.TempDir() |
| - if _, _, err := Restore(ctx, s, pfx, dir); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err != nil { |
| t.Fatal(err) |
| } |
| fi, err := os.Stat(filepath.Join(dir, "x")) |
| @@ -573,7 +573,7 @@ func TestOverlappingLiveEntriesKeepTheNewerOne(t *testing.T) { |
| stage(t, p, "a", "file") // newer than the child |
| commit(t, p) |
| dir := t.TempDir() |
| - _, st, err := Restore(ctx, s, pfx, dir) |
| + _, st, err := Restore(ctx, s, pfx, dir, nil) |
| if err != nil { |
| t.Fatal(err) |
| } |
| @@ -586,7 +586,7 @@ func TestOverlappingLiveEntriesKeepTheNewerOne(t *testing.T) { |
| stage(t, q, "a/b", "child") // newer than the file |
| commit(t, q) |
| dir2 := t.TempDir() |
| - _, st, err = Restore(ctx, q.s, pfx, dir2) |
| + _, st, err = Restore(ctx, q.s, pfx, dir2, nil) |
| if err != nil { |
| t.Fatal(err) |
| } |
| @@ -621,7 +621,7 @@ func TestThreeDeepOverlapKeepsTheNewestEntry(t *testing.T) { |
| stage(t, p, "a/b", "two") // the newest |
| commit(t, p) |
| dir := t.TempDir() |
| - _, st, err := Restore(ctx, s, pfx, dir) |
| + _, st, err := Restore(ctx, s, pfx, dir, nil) |
| if err != nil { |
| t.Fatal(err) |
| } |
| @@ -719,22 +719,22 @@ func TestRestoreRefusesAnEntryWhoseObjectOrHashIsMalformedAndTouchesNothing(t *t |
| "uppercase sha": func(m *Manifest) { |
| e := m.Entries["a"] |
| e.SHA256 = strings.ToUpper(e.SHA256) |
| - e.Object = objectKey(e.Seq, e.SHA256) |
| + e.Object = pfx + objectKey(e.Seq, e.SHA256) |
| m.Entries["a"] = e |
| }, |
| "short sha": func(m *Manifest) { |
| e := m.Entries["a"] |
| - e.SHA256, e.Object = "aa", objectKey(e.Seq, "aa") |
| + e.SHA256, e.Object = "aa", pfx+objectKey(e.Seq, "aa") |
| m.Entries["a"] = e |
| }, |
| "object elsewhere": func(m *Manifest) { |
| e := m.Entries["a"] |
| - e.Object = "../../other/files/g1/" + e.Object |
| + e.Object = "instances/other/files/g1/" + objectKey(e.Seq, e.SHA256) |
| m.Entries["a"] = e |
| }, |
| "object with another seq": func(m *Manifest) { |
| e := m.Entries["a"] |
| - e.Object = objectKey(e.Seq+1, e.SHA256) |
| + e.Object = pfx + objectKey(e.Seq+1, e.SHA256) |
| m.Entries["a"] = e |
| }, |
| "platform object elsewhere": func(m *Manifest) { |
| @@ -745,13 +745,13 @@ func TestRestoreRefusesAnEntryWhoseObjectOrHashIsMalformedAndTouchesNothing(t *t |
| "platform sha not hex": func(m *Manifest) { |
| e := m.Platform["item"] |
| e.SHA256 = strings.Repeat("g", 64) |
| - e.Object = objectKey(e.Seq, e.SHA256) |
| + e.Object = pfx + objectKey(e.Seq, e.SHA256) |
| m.Platform["item"] = e |
| }, |
| } |
| for name, bend := range cases { |
| s, dir := putHostile(t, bend) |
| - if _, _, err := Restore(context.Background(), s, pfx, dir); err == nil { |
| + if _, _, err := Restore(context.Background(), s, pfx, dir, nil); err == nil { |
| t.Fatalf("%s: Restore accepted the manifest", name) |
| } |
| if got := read(t, dir, "gone"); got != "still here" { |
| @@ -766,7 +766,7 @@ func TestRestoreRefusesAnEntryWhoseObjectOrHashIsMalformedAndTouchesNothing(t *t |
| func TestReadPlatformRefusesAMalformedEntry(t *testing.T) { |
| s, _ := putHostile(t, func(m *Manifest) { |
| e := m.Platform["item"] |
| - e.Object = "objects/elsewhere" |
| + e.Object = pfx + "objects/elsewhere" |
| m.Platform["item"] = e |
| }) |
| ctx := context.Background() |
| @@ -778,7 +778,7 @@ func TestReadPlatformRefusesAMalformedEntry(t *testing.T) { |
| if err != nil { |
| t.Fatal(err) |
| } |
| - if _, err := ReadPlatform(ctx, s, pfx, m, "item"); err == nil { |
| + if _, err := ReadPlatform(ctx, s, pfx, m, "item", nil); err == nil { |
| t.Fatal("ReadPlatform accepted an object key that is not its own") |
| } |
| } |
| @@ -790,7 +790,7 @@ func TestRestoreRefusesAnEntryWhoseSizeIsOutOfRangeAndTouchesNothing(t *testing. |
| e.Size = size |
| m.Entries["a"] = e |
| }) |
| - if _, _, err := Restore(context.Background(), s, pfx, dir); err == nil { |
| + if _, _, err := Restore(context.Background(), s, pfx, dir, nil); err == nil { |
| t.Fatalf("size %d: Restore accepted the manifest", size) |
| } |
| if got := read(t, dir, "gone"); got != "still here" { |
| @@ -809,14 +809,14 @@ func TestFetchRefusesAnObjectWhoseLengthIsNotTheRecordedSize(t *testing.T) { |
| m.Platform["item"] = p |
| }) |
| ctx := context.Background() |
| - if _, _, err := Restore(ctx, s, pfx, dir); err == nil || !strings.Contains(err.Error(), "size") { |
| + if _, _, err := Restore(ctx, s, pfx, dir, nil); err == nil || !strings.Contains(err.Error(), "size") { |
| t.Fatalf("Restore err = %v, want a size mismatch", err) |
| } |
| m, _, err := Load(ctx, s, pfx) |
| if err != nil { |
| t.Fatal(err) |
| } |
| - if _, err := ReadPlatform(ctx, s, pfx, m, "item"); err == nil || !strings.Contains(err.Error(), "size") { |
| + if _, err := ReadPlatform(ctx, s, pfx, m, "item", nil); err == nil || !strings.Contains(err.Error(), "size") { |
| t.Fatalf("ReadPlatform err = %v, want a size mismatch", err) |
| } |
| } |
diff --git a/internal/hostfiles/manifest.go b/internal/hostfiles/manifest.go| index 80111a6ee..bf441538a 100644 |
| --- a/internal/hostfiles/manifest.go |
| +++ b/internal/hostfiles/manifest.go |
| @@ -23,6 +23,13 @@ import ( |
| // rather than read. Plan 1b lifts this with a streaming put. |
| const MaxFileBytes = 64 << 20 |
| +// Entry is one path's state. Object is origin-qualified: the object's full |
| +// key from the bucket root, "<origin>objects/<seq>-<sha256>", where origin |
| +// is the files prefix of the generation that wrote it. A manifest copied |
| +// into a later generation keeps every Object as it was, so the copy names |
| +// the same immutable objects and nothing is uploaded again (claim L13). |
| +// Readers resolve Object exactly as written, never against their own |
| +// prefix, and only within the origins their caller admits. |
| type Entry struct { |
| Object string `json:"object,omitempty"` |
| SHA256 string `json:"sha256,omitempty"` |
| @@ -34,6 +41,19 @@ type Entry struct { |
| Skipped string `json:"skipped,omitempty"` |
| } |
| +// ManifestVersion is the manifest format this binary reads and writes. |
| +// |
| +// Version 2 made Entry.Object origin-qualified (a full key) where version |
| +// 1 held a key relative to the manifest's prefix. That changes what an |
| +// existing field means; it adds nothing a reader could ignore. A version 1 |
| +// reader would join a qualified key to its own prefix and look in the |
| +// wrong place, and a version 1 manifest copied to another generation |
| +// would resolve against the copy's prefix. So the version is bumped and |
| +// every other version is refused. No version 1 manifest was written in |
| +// production (nothing called the publisher), so there is nothing to |
| +// migrate. |
| +const ManifestVersion = 2 |
| + |
| type Manifest struct { |
| V int `json:"v"` |
| Seq int64 `json:"seq"` |
| @@ -43,8 +63,40 @@ type Manifest struct { |
| func ManifestKey(prefix string) string { return prefix + "manifest.json" } |
| +// objectKey is an object's key relative to the origin that wrote it. |
| func objectKey(seq int64, sha string) string { return fmt.Sprintf("objects/%016d-%s", seq, sha) } |
| +// ErrOrigin marks an entry whose object lies outside the origins its |
| +// reader admits, or an origin that is not a well-formed prefix. |
| +var ErrOrigin = errors.New("hostfiles: object origin not admitted") |
| + |
| +// checkPrefix accepts a generation prefix: a clean, relative bucket key |
| +// ending in "/". |
| +func checkPrefix(prefix string) error { |
| + dir, ok := strings.CutSuffix(prefix, "/") |
| + if !ok { |
| + return fmt.Errorf("hostfiles: prefix %q must end in /", prefix) |
| + } |
| + if _, err := CleanPath(dir); err != nil { |
| + return fmt.Errorf("hostfiles: prefix %q is not a clean bucket key", prefix) |
| + } |
| + return nil |
| +} |
| + |
| +// admitted resolves a caller's origins: nil or empty means the manifest's |
| +// own prefix alone. Every origin must be a well-formed prefix. |
| +func admitted(prefix string, origins []string) ([]string, error) { |
| + if len(origins) == 0 { |
| + origins = []string{prefix} |
| + } |
| + for _, o := range origins { |
| + if err := checkPrefix(o); err != nil { |
| + return nil, fmt.Errorf("%w: %v", ErrOrigin, err) |
| + } |
| + } |
| + return origins, nil |
| +} |
| + |
| // CleanPath accepts only a relative, already-clean, slash-separated path |
| // that stays inside its root. It never repairs a path: a manifest is |
| // written by this package, so anything else is damage or an attack. |
| @@ -66,7 +118,7 @@ func Load(ctx context.Context, s release.Store, prefix string) (Manifest, bool, |
| } |
| func load(ctx context.Context, s release.Store, prefix string) (Manifest, string, bool, error) { |
| - empty := Manifest{V: 1, Entries: map[string]Entry{}, Platform: map[string]Entry{}} |
| + empty := Manifest{V: ManifestVersion, Entries: map[string]Entry{}, Platform: map[string]Entry{}} |
| raw, etag, err := s.Get(ctx, ManifestKey(prefix)) |
| if errors.Is(err, os.ErrNotExist) { |
| return empty, "", false, nil |
| @@ -78,8 +130,8 @@ func load(ctx context.Context, s release.Store, prefix string) (Manifest, string |
| if err := json.Unmarshal(raw, &m); err != nil { |
| return Manifest{}, "", false, fmt.Errorf("hostfiles: manifest %s: %w", prefix, err) |
| } |
| - if m.V != 1 { |
| - return Manifest{}, "", false, fmt.Errorf("hostfiles: manifest %s is version %d; this binary reads 1", prefix, m.V) |
| + if m.V != ManifestVersion { |
| + return Manifest{}, "", false, fmt.Errorf("hostfiles: manifest %s is version %d; this binary reads %d", prefix, m.V, ManifestVersion) |
| } |
| if m.Entries == nil { |
| m.Entries = map[string]Entry{} |
diff --git a/internal/hostfiles/origins_test.go b/internal/hostfiles/origins_test.go| new file mode 100644 |
| index 000000000..f9e0d1653 |
| --- /dev/null |
| +++ b/internal/hostfiles/origins_test.go |
| @@ -0,0 +1,304 @@ |
| +package hostfiles |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// writeFile is sync_test.go's write, which is unix-only. |
| +func writeFile(t *testing.T, dir, rel, body string) { |
| + t.Helper() |
| + full := filepath.Join(dir, rel) |
| + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(full, []byte(body), 0o640); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// pfx2 is the next generation of pfx's incarnation. |
| +const pfx2 = "instances/shop.test/abc/files/g2/" |
| + |
| +// publishG1 publishes two files and a platform item under pfx. |
| +func publishG1(t *testing.T, s release.Store) { |
| + t.Helper() |
| + p := open(t, s) |
| + stage(t, p, "instance.key", "k1") |
| + stage(t, p, "uploads/a/b.txt", "hello") |
| + if err := p.StagePlatform(context.Background(), "env", []byte("A=1")); err != nil { |
| + t.Fatal(err) |
| + } |
| + commit(t, p) |
| +} |
| + |
| +func TestEntriesCarryTheirOriginQualifiedObjectKey(t *testing.T) { |
| + ctx, s := context.Background(), release.NewDirStore(t.TempDir()) |
| + publishG1(t, s) |
| + m, _, err := Load(ctx, s, pfx) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if m.V != 2 { |
| + t.Fatalf("manifest version = %d, want 2", m.V) |
| + } |
| + e := m.Entries["instance.key"] |
| + if want := pfx + objectKey(e.Seq, e.SHA256); e.Object != want { |
| + t.Fatalf("object = %q, want %q", e.Object, want) |
| + } |
| + if _, _, err := s.Get(ctx, e.Object); err != nil { |
| + t.Fatalf("the object is not at its key: %v", err) |
| + } |
| +} |
| + |
| +func TestACopiedManifestRestoresTheSameBytesWithoutCopyingObjects(t *testing.T) { |
| + ctx, s := context.Background(), release.NewDirStore(t.TempDir()) |
| + publishG1(t, s) |
| + m, err := CopyManifest(ctx, s, pfx, pfx2, nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if keys, _ := s.List(ctx, pfx2+"objects/"); len(keys) != 0 { |
| + t.Fatalf("the copy wrote objects: %v", keys) |
| + } |
| + src, _, _ := Load(ctx, s, pfx) |
| + if m.Seq != src.Seq || len(m.Entries) != len(src.Entries) { |
| + t.Fatalf("copy = %+v, source = %+v", m, src) |
| + } |
| + dir := t.TempDir() |
| + if _, _, err := Restore(ctx, s, pfx2, dir, []string{pfx, pfx2}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if read(t, dir, "instance.key") != "k1" || read(t, dir, "uploads/a/b.txt") != "hello" { |
| + t.Fatal("restored the wrong bytes") |
| + } |
| + got, err := ReadPlatform(ctx, s, pfx2, m, "env", []string{pfx, pfx2}) |
| + if err != nil || string(got) != "A=1" { |
| + t.Fatalf("ReadPlatform = %q, %v", got, err) |
| + } |
| + |
| + // The new generation's publisher continues the sequence. A changed file |
| + // is a new object under g2, an unchanged one keeps its g1 reference. |
| + p, err := Open(ctx, s, pfx2) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + stage(t, p, "instance.key", "k1") |
| + stage(t, p, "uploads/a/b.txt", "changed") |
| + if seq := commit(t, p); seq != src.Seq+1 { |
| + t.Fatalf("seq = %d, want %d", seq, src.Seq+1) |
| + } |
| + m2, _, _ := Load(ctx, s, pfx2) |
| + if !strings.HasPrefix(m2.Entries["instance.key"].Object, pfx) || !strings.HasPrefix(m2.Entries["uploads/a/b.txt"].Object, pfx2) { |
| + t.Fatalf("entries = %+v", m2.Entries) |
| + } |
| + dir2 := t.TempDir() |
| + if _, _, err := Restore(ctx, s, pfx2, dir2, []string{pfx, pfx2}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if read(t, dir2, "instance.key") != "k1" || read(t, dir2, "uploads/a/b.txt") != "changed" { |
| + t.Fatal("restored the wrong bytes after the second generation published") |
| + } |
| + // The source generation is untouched. |
| + if again, _, _ := Load(ctx, s, pfx); again.Seq != src.Seq { |
| + t.Fatalf("source seq moved to %d", again.Seq) |
| + } |
| +} |
| + |
| +func TestRestoreRefusesAnEntryOutsideTheAllowedOriginsAndTouchesNothing(t *testing.T) { |
| + ctx, s := context.Background(), release.NewDirStore(t.TempDir()) |
| + publishG1(t, s) |
| + m, err := CopyManifest(ctx, s, pfx, pfx2, nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for name, origins := range map[string][]string{ |
| + "default (own prefix only)": nil, |
| + "another generation only": {"instances/shop.test/abc/files/g0/", pfx2}, |
| + } { |
| + dir := t.TempDir() |
| + if err := os.WriteFile(filepath.Join(dir, "instance.key"), []byte("before"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, _, err := Restore(ctx, s, pfx2, dir, origins); err == nil || !strings.Contains(err.Error(), "origin") { |
| + t.Fatalf("%s: Restore err = %v, want an origin refusal", name, err) |
| + } |
| + if read(t, dir, "instance.key") != "before" { |
| + t.Fatalf("%s: Restore wrote before refusing", name) |
| + } |
| + if _, err := os.Stat(filepath.Join(dir, "uploads")); !os.IsNotExist(err) { |
| + t.Fatalf("%s: Restore created uploads/", name) |
| + } |
| + if _, err := ReadPlatform(ctx, s, pfx2, m, "env", origins); err == nil { |
| + t.Fatalf("%s: ReadPlatform read an object outside its origins", name) |
| + } |
| + } |
| +} |
| + |
| +func TestRestoreRefusesATamperedObjectKey(t *testing.T) { |
| + rel := func(e Entry) string { return objectKey(e.Seq, e.SHA256) } |
| + cases := map[string]func(e Entry) string{ |
| + "wrong seq": func(e Entry) string { return pfx + objectKey(e.Seq+1, e.SHA256) }, |
| + "wrong sha": func(e Entry) string { return pfx + objectKey(e.Seq, strings.Repeat("0", 64)) }, |
| + "relative (version 1)": rel, |
| + "dot-dot to a sibling gen": func(e Entry) string { return pfx + "../g9/" + rel(e) }, |
| + "dot-dot inside": func(e Entry) string { return pfx + "objects/../" + rel(e) }, |
| + "absolute": func(e Entry) string { return "/" + pfx + rel(e) }, |
| + "another host's prefix": func(e Entry) string { return "instances/other.test/abc/files/g1/" + rel(e) }, |
| + "nested under the origin": func(e Entry) string { return pfx + "x/" + rel(e) }, |
| + "double slash": func(e Entry) string { return strings.TrimSuffix(pfx, "/") + "//" + rel(e) }, |
| + "empty": func(Entry) string { return "" }, |
| + } |
| + for name, key := range cases { |
| + for _, platform := range []bool{false, true} { |
| + s, dir := putHostile(t, func(m *Manifest) { |
| + if platform { |
| + e := m.Platform["item"] |
| + e.Object = key(e) |
| + m.Platform["item"] = e |
| + return |
| + } |
| + e := m.Entries["a"] |
| + e.Object = key(e) |
| + m.Entries["a"] = e |
| + }) |
| + if _, _, err := Restore(context.Background(), s, pfx, dir, nil); err == nil { |
| + t.Fatalf("%s (platform %v): Restore accepted the manifest", name, platform) |
| + } |
| + if got := read(t, dir, "gone"); got != "still here" { |
| + t.Fatalf("%s (platform %v): gone = %q, want it untouched", name, platform, got) |
| + } |
| + if _, err := os.Stat(filepath.Join(dir, "a")); !os.IsNotExist(err) { |
| + t.Fatalf("%s (platform %v): a was written", name, platform) |
| + } |
| + } |
| + } |
| +} |
| + |
| +func TestRestoreRefusesAMalformedOrigin(t *testing.T) { |
| + ctx, s := context.Background(), release.NewDirStore(t.TempDir()) |
| + publishG1(t, s) |
| + for _, o := range []string{"", "/", "instances/shop.test/abc/files/g1", "/instances/shop.test/abc/files/g1/", |
| + "instances/shop.test/abc/files/../files/g1/", "instances//shop.test/", "./instances/"} { |
| + if _, _, err := Restore(ctx, s, pfx, t.TempDir(), []string{o}); err == nil { |
| + t.Fatalf("Restore accepted origin %q", o) |
| + } |
| + } |
| +} |
| + |
| +func TestAManifestOfAnotherVersionIsRefused(t *testing.T) { |
| + for _, v := range []int{0, 1, 3} { |
| + ctx, s := context.Background(), release.NewDirStore(t.TempDir()) |
| + body, _ := json.Marshal(map[string]any{"v": v, "seq": 0, "entries": map[string]any{}, "platform": map[string]any{}}) |
| + if _, err := s.Put(ctx, ManifestKey(pfx), body, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, _, err := Load(ctx, s, pfx); err == nil || !strings.Contains(err.Error(), "version") { |
| + t.Fatalf("v%d: Load err = %v", v, err) |
| + } |
| + if _, err := Open(ctx, s, pfx); err == nil { |
| + t.Fatalf("v%d: Open accepted it", v) |
| + } |
| + if _, _, err := Restore(ctx, s, pfx, t.TempDir(), nil); err == nil { |
| + t.Fatalf("v%d: Restore accepted it", v) |
| + } |
| + if _, err := CopyManifest(ctx, s, pfx, pfx2, nil); err == nil { |
| + t.Fatalf("v%d: CopyManifest accepted it", v) |
| + } |
| + } |
| +} |
| + |
| +func TestCopyManifestIsCreateOnlyAndRetrySafe(t *testing.T) { |
| + ctx, s := context.Background(), release.NewDirStore(t.TempDir()) |
| + publishG1(t, s) |
| + if _, err := CopyManifest(ctx, s, pfx, pfx2, nil); err != nil { |
| + t.Fatal(err) |
| + } |
| + // A repeat whose first response was lost finds its own bytes: success. |
| + if _, err := CopyManifest(ctx, s, pfx, pfx2, nil); err != nil { |
| + t.Fatalf("a repeated copy: %v", err) |
| + } |
| + // Once g2 has moved on, a copy is refused and changes nothing. |
| + p, err := Open(ctx, s, pfx2) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + stage(t, p, "new", "n") |
| + commit(t, p) |
| + before, _, _ := s.Get(ctx, ManifestKey(pfx2)) |
| + if _, err := CopyManifest(ctx, s, pfx, pfx2, nil); !errors.Is(err, ErrManifestExists) { |
| + t.Fatalf("err = %v, want ErrManifestExists", err) |
| + } |
| + after, _, _ := s.Get(ctx, ManifestKey(pfx2)) |
| + if string(before) != string(after) { |
| + t.Fatal("a refused copy changed the target") |
| + } |
| +} |
| + |
| +func TestCopyManifestRefusesAMissingOrInvalidSourceAndWritesNothing(t *testing.T) { |
| + ctx, s := context.Background(), release.NewDirStore(t.TempDir()) |
| + if _, err := CopyManifest(ctx, s, pfx, pfx2, nil); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("absent source: err = %v, want os.ErrNotExist", err) |
| + } |
| + // g2 copied from g1 references g1; copying g2 on to g3 with only g2 |
| + // admitted is refused. |
| + publishG1(t, s) |
| + if _, err := CopyManifest(ctx, s, pfx, pfx2, nil); err != nil { |
| + t.Fatal(err) |
| + } |
| + const pfx3 = "instances/shop.test/abc/files/g3/" |
| + if _, err := CopyManifest(ctx, s, pfx2, pfx3, nil); err == nil || !strings.Contains(err.Error(), "origin") { |
| + t.Fatalf("err = %v, want an origin refusal", err) |
| + } |
| + if _, _, err := s.Get(ctx, ManifestKey(pfx3)); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("a refused copy wrote a manifest: %v", err) |
| + } |
| + if _, err := CopyManifest(ctx, s, pfx2, pfx3, []string{pfx, pfx2}); err != nil { |
| + t.Fatalf("with g1 admitted: %v", err) |
| + } |
| + for _, bad := range [][2]string{{pfx, pfx}, {pfx, "instances/shop.test/abc/files/g4"}, {"nope", pfx2}} { |
| + if _, err := CopyManifest(ctx, s, bad[0], bad[1], nil); err == nil { |
| + t.Fatalf("CopyManifest(%q, %q) accepted", bad[0], bad[1]) |
| + } |
| + } |
| +} |
| + |
| +func TestPublishCommitsTheDirectoryAsOneRevision(t *testing.T) { |
| + ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir() |
| + writeFile(t, src, "instance.key", "k1") |
| + writeFile(t, src, "uploads/a.txt", "a") |
| + writeFile(t, src, "skip/me", "x") |
| + skip := func(rel string, isDir bool) bool { return rel == "skip" } |
| + seq, st, err := Publish(ctx, s, pfx, src, skip) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if seq != 2 || st.Published != 2 { |
| + t.Fatalf("seq = %d, stats = %+v", seq, st) |
| + } |
| + dir := t.TempDir() |
| + m, _, err := Restore(ctx, s, pfx, dir, nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if m.Seq != seq || read(t, dir, "instance.key") != "k1" || read(t, dir, "uploads/a.txt") != "a" { |
| + t.Fatalf("m = %+v", m) |
| + } |
| + if _, ok := m.Entries["skip/me"]; ok { |
| + t.Fatal("a skipped path was published") |
| + } |
| + // Publishing again with nothing changed is the same revision. |
| + if again, _, err := Publish(ctx, s, pfx, src, skip); err != nil || again != seq { |
| + t.Fatalf("again = %d, %v", again, err) |
| + } |
| + if _, _, err := Publish(ctx, s, pfx, filepath.Join(src, "absent"), nil); err == nil { |
| + t.Fatal("published a missing directory") |
| + } |
| +} |
diff --git a/internal/hostfiles/publisher.go b/internal/hostfiles/publisher.go| index b1708f891..a55621888 100644 |
| --- a/internal/hostfiles/publisher.go |
| +++ b/internal/hostfiles/publisher.go |
| @@ -39,8 +39,8 @@ type Publisher struct { |
| } |
| func Open(ctx context.Context, s release.Store, prefix string) (*Publisher, error) { |
| - if !strings.HasSuffix(prefix, "/") { |
| - return nil, fmt.Errorf("hostfiles: prefix %q must end in /", prefix) |
| + if err := checkPrefix(prefix); err != nil { |
| + return nil, err |
| } |
| m, etag, _, err := load(ctx, s, prefix) |
| if err != nil { |
| @@ -54,13 +54,14 @@ func sum(body []byte) string { |
| return hex.EncodeToString(h[:]) |
| } |
| -// put writes body as the next immutable object and returns its entry. |
| -// Callers hold p.mu. |
| +// put writes body as the next immutable object under this publisher's own |
| +// prefix and returns its entry, whose Object is the full key. Callers hold |
| +// p.mu. |
| func (p *Publisher) put(ctx context.Context, body []byte, mode uint32, torn bool) (Entry, error) { |
| sha := sum(body) |
| seq := p.next + 1 |
| - key := objectKey(seq, sha) |
| - if _, err := p.s.Put(ctx, p.prefix+key, body, ""); err != nil && !errors.Is(err, release.ErrCASConflict) { |
| + key := p.prefix + objectKey(seq, sha) |
| + if _, err := p.s.Put(ctx, key, body, ""); err != nil && !errors.Is(err, release.ErrCASConflict) { |
| // A conflict means this exact key exists: the same sequence number |
| // and the same hash, left by an attempt that crashed before its |
| // commit. The bytes are the ones we were about to write. |
diff --git a/internal/hostfiles/restore.go b/internal/hostfiles/restore.go| index 03da182ce..736b59260 100644 |
| --- a/internal/hostfiles/restore.go |
| +++ b/internal/hostfiles/restore.go |
| @@ -10,6 +10,7 @@ import ( |
| "os" |
| "path" |
| "path/filepath" |
| + "slices" |
| "sort" |
| "strings" |
| @@ -48,25 +49,28 @@ type RestoreStats struct { |
| // before its rename, and every directory in which a name was created, |
| // replaced or removed has been synced since. A caller may record the |
| // installation as complete on that basis, and on no weaker one. |
| -func Restore(ctx context.Context, s release.Store, prefix, dir string) (Manifest, RestoreStats, error) { |
| +// |
| +// origins are the generation prefixes whose objects this restore may read: |
| +// the set the caller's pin admits. nil or empty means prefix alone. Every |
| +// entry's object must be exactly "<origin>objects/<seq>-<sha256>" for one |
| +// of them, with the entry's own Seq and SHA256; any other entry refuses the |
| +// whole manifest before anything is touched (ErrOrigin when the key is |
| +// well formed but its origin is not admitted). |
| +func Restore(ctx context.Context, s release.Store, prefix, dir string, origins []string) (Manifest, RestoreStats, error) { |
| var st RestoreStats |
| + origins, err := admitted(prefix, origins) |
| + if err != nil { |
| + return Manifest{}, st, err |
| + } |
| m, _, _, err := load(ctx, s, prefix) |
| if err != nil { |
| return Manifest{}, st, err |
| } |
| - var live, dead []string |
| - for name, e := range m.Platform { |
| - if err := checkEntry(e); err != nil { |
| - return Manifest{}, st, fmt.Errorf("hostfiles: platform item %q: %w", name, err) |
| - } |
| + if err := checkManifest(m, origins); err != nil { |
| + return Manifest{}, st, err |
| } |
| + var live, dead []string |
| for p, e := range m.Entries { |
| - if _, err := CleanPath(p); err != nil { |
| - return Manifest{}, st, err |
| - } |
| - if err := checkEntry(e); err != nil { |
| - return Manifest{}, st, fmt.Errorf("hostfiles: %s: %w", p, err) |
| - } |
| if e.Deleted { |
| dead = append(dead, p) |
| } else { |
| @@ -147,7 +151,7 @@ func Restore(ctx context.Context, s release.Store, prefix, dir string) (Manifest |
| continue |
| } |
| mark(p) |
| - body, err := fetch(ctx, s, prefix, e) |
| + body, err := fetch(ctx, s, e, origins) |
| if err != nil { |
| return Manifest{}, st, fmt.Errorf("hostfiles: %s: %w", p, err) |
| } |
| @@ -191,27 +195,60 @@ func Restore(ctx context.Context, s release.Store, prefix, dir string) (Manifest |
| } |
| // ReadPlatform returns a platform-managed item's bytes, verified. A name |
| -// the manifest does not hold is os.ErrNotExist. |
| -func ReadPlatform(ctx context.Context, s release.Store, prefix string, m Manifest, name string) ([]byte, error) { |
| +// the manifest does not hold is os.ErrNotExist. prefix is where m was |
| +// loaded from; origins are as for Restore (nil or empty: prefix alone). |
| +func ReadPlatform(ctx context.Context, s release.Store, prefix string, m Manifest, name string, origins []string) ([]byte, error) { |
| + origins, err := admitted(prefix, origins) |
| + if err != nil { |
| + return nil, err |
| + } |
| e, ok := m.Platform[name] |
| if !ok { |
| return nil, fmt.Errorf("hostfiles: platform item %q: %w", name, os.ErrNotExist) |
| } |
| - return fetch(ctx, s, prefix, e) |
| + return fetch(ctx, s, e, origins) |
| } |
| -// checkEntry refuses a published entry whose object is not the one this |
| -// package would have written for it, or whose size it could never have |
| -// published. Tombstones and skipped entries name no object. |
| -func checkEntry(e Entry) error { |
| +// checkManifest validates every path and every entry before a caller acts |
| +// on any of them. |
| +func checkManifest(m Manifest, origins []string) error { |
| + for name, e := range m.Platform { |
| + if err := checkEntry(e, origins); err != nil { |
| + return fmt.Errorf("hostfiles: platform item %q: %w", name, err) |
| + } |
| + } |
| + for p, e := range m.Entries { |
| + if _, err := CleanPath(p); err != nil { |
| + return err |
| + } |
| + if err := checkEntry(e, origins); err != nil { |
| + return fmt.Errorf("hostfiles: %s: %w", p, err) |
| + } |
| + } |
| + return nil |
| +} |
| + |
| +// checkEntry refuses a published entry whose object is not one this |
| +// package would have written for it in an admitted origin, or whose size |
| +// it could never have published. Tombstones and skipped entries name no |
| +// object. origins are already validated prefixes. |
| +func checkEntry(e Entry, origins []string) error { |
| if e.Deleted || e.Skipped != "" { |
| return nil |
| } |
| if len(e.SHA256) != 64 || strings.Trim(e.SHA256, "0123456789abcdef") != "" { |
| return fmt.Errorf("sha256 %q is not 64 lowercase hex characters", e.SHA256) |
| } |
| - if want := objectKey(e.Seq, e.SHA256); e.Object != want { |
| - return fmt.Errorf("object %q is not %q", e.Object, want) |
| + if _, err := CleanPath(e.Object); err != nil { |
| + return fmt.Errorf("object %q is not a clean bucket key", e.Object) |
| + } |
| + want := objectKey(e.Seq, e.SHA256) |
| + origin, ok := strings.CutSuffix(e.Object, want) |
| + if !ok { |
| + return fmt.Errorf("object %q does not end in %q", e.Object, want) |
| + } |
| + if !slices.Contains(origins, origin) { |
| + return fmt.Errorf("%w: object %q lies under %q, not one of %q", ErrOrigin, e.Object, origin, origins) |
| } |
| if e.Size < 0 || e.Size > MaxFileBytes { |
| return fmt.Errorf("size %d is outside 0 to %d", e.Size, MaxFileBytes) |
| @@ -219,11 +256,12 @@ func checkEntry(e Entry) error { |
| return nil |
| } |
| -func fetch(ctx context.Context, s release.Store, prefix string, e Entry) ([]byte, error) { |
| - if err := checkEntry(e); err != nil { |
| +// fetch reads e's object at exactly the key the entry names. |
| +func fetch(ctx context.Context, s release.Store, e Entry, origins []string) ([]byte, error) { |
| + if err := checkEntry(e, origins); err != nil { |
| return nil, err |
| } |
| - body, _, err := s.Get(ctx, prefix+e.Object) |
| + body, _, err := s.Get(ctx, e.Object) |
| if err != nil { |
| return nil, err |
| } |
diff --git a/internal/hostfiles/sync_test.go b/internal/hostfiles/sync_test.go| index 9d79c771f..8a767fe22 100644 |
| --- a/internal/hostfiles/sync_test.go |
| +++ b/internal/hostfiles/sync_test.go |
| @@ -47,7 +47,7 @@ func TestSyncDirPublishesNewChangedAndDeleted(t *testing.T) { |
| commit(t, p) |
| dst := t.TempDir() |
| - if _, _, err := Restore(ctx, s, pfx, dst); err != nil { |
| + if _, _, err := Restore(ctx, s, pfx, dst, nil); err != nil { |
| t.Fatal(err) |
| } |
| if read(t, dst, "uploads/a.txt") != "a2" || read(t, dst, "new.txt") != "n" { |
| @@ -131,7 +131,7 @@ func TestSyncDirRecordsAFileOverTheCapWithoutReadingIt(t *testing.T) { |
| t.Fatalf("the oversized file was read %d times; it must be refused on its size alone", reads) |
| } |
| commit(t, p) |
| - _, rs, err := Restore(ctx, s, pfx, t.TempDir()) |
| + _, rs, err := Restore(ctx, s, pfx, t.TempDir(), nil) |
| if err != nil || !reflect.DeepEqual(rs.Skipped, []string{"huge.bin"}) { |
| t.Fatalf("restore skipped %v, err %v", rs.Skipped, err) |
| } |
diff --git a/internal/hostfiles/sync_unix_test.go b/internal/hostfiles/sync_unix_test.go| index c9c9c974a..c811cb715 100644 |
| --- a/internal/hostfiles/sync_unix_test.go |
| +++ b/internal/hostfiles/sync_unix_test.go |
| @@ -252,3 +252,23 @@ func TestSyncDirKeepsAnEntryWhoseFileWasReplacedMidLook(t *testing.T) { |
| t.Fatalf("the next scan did not pick the replacement up: dirty = %v err = %v", p.Dirty(), err) |
| } |
| } |
| + |
| +// Publish is an installation's record of its directory: a file it could |
| +// not read would be missing from it, so it commits nothing and says so. |
| +func TestPublishRefusesADirectoryWithAnUnreadableFile(t *testing.T) { |
| + if os.Getuid() == 0 { |
| + t.Skip("root can read anything") |
| + } |
| + ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir() |
| + write(t, src, "ok", "v1") |
| + write(t, src, "locked", "v1") |
| + if err := os.Chmod(filepath.Join(src, "locked"), 0); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, st, err := Publish(ctx, s, pfx, src, nil); err == nil || !reflect.DeepEqual(st.Unreadable, []string{"locked"}) { |
| + t.Fatalf("err = %v, stats = %+v", err, st) |
| + } |
| + if _, found, err := Load(ctx, s, pfx); found || err != nil { |
| + t.Fatalf("a manifest was committed: found = %v, err = %v", found, err) |
| + } |
| +} |
diff --git a/internal/hostfiles/wake.go b/internal/hostfiles/wake.go| index fae6854da..e603ca023 100644 |
| --- a/internal/hostfiles/wake.go |
| +++ b/internal/hostfiles/wake.go |
| @@ -121,12 +121,12 @@ func StageWake(ctx context.Context, p *Publisher, ws WakeState) error { |
| // when the manifest has no wake entry at all. A manifest that names wake |
| // state whose object is missing or damaged is an error: that is lost |
| // timers, not a host with none, and the caller must not install as if it |
| -// were. |
| -func LoadWake(ctx context.Context, s release.Store, prefix string, m Manifest) (WakeState, bool, error) { |
| +// were. origins are as for ReadPlatform. |
| +func LoadWake(ctx context.Context, s release.Store, prefix string, m Manifest, origins []string) (WakeState, bool, error) { |
| if _, ok := m.Platform[WakeItem]; !ok { |
| return WakeState{}, false, nil |
| } |
| - body, err := ReadPlatform(ctx, s, prefix, m, WakeItem) |
| + body, err := ReadPlatform(ctx, s, prefix, m, WakeItem, origins) |
| if err != nil { |
| return WakeState{}, false, fmt.Errorf("hostfiles: the manifest names wake state that cannot be read: %w", err) |
| } |
diff --git a/internal/hostfiles/wake_test.go b/internal/hostfiles/wake_test.go| index 435d1dc1b..aef0a715a 100644 |
| --- a/internal/hostfiles/wake_test.go |
| +++ b/internal/hostfiles/wake_test.go |
| @@ -63,7 +63,7 @@ func TestWakeStateRoundTripsThroughTheManifest(t *testing.T) { |
| if err != nil { |
| t.Fatal(err) |
| } |
| - got, found, err := LoadWake(ctx, s, pfx, m) |
| + got, found, err := LoadWake(ctx, s, pfx, m, nil) |
| if err != nil || !found { |
| t.Fatalf("LoadWake found=%v err=%v", found, err) |
| } |
| @@ -102,7 +102,7 @@ func TestStageWakeOfUnchangedStateStagesNothing(t *testing.T) { |
| func TestLoadWakeWithNoneReportsNotFound(t *testing.T) { |
| ctx, s := context.Background(), release.NewDirStore(t.TempDir()) |
| m, _, _ := Load(ctx, s, pfx) |
| - if _, found, err := LoadWake(ctx, s, pfx, m); found || err != nil { |
| + if _, found, err := LoadWake(ctx, s, pfx, m, nil); found || err != nil { |
| t.Fatalf("found=%v err=%v", found, err) |
| } |
| } |
| @@ -120,10 +120,10 @@ func TestLoadWakeWithItsObjectMissingIsAnError(t *testing.T) { |
| if err != nil { |
| t.Fatal(err) |
| } |
| - if err := s.Delete(ctx, pfx+m.Platform[WakeItem].Object); err != nil { |
| + if err := s.Delete(ctx, m.Platform[WakeItem].Object); err != nil { |
| t.Fatal(err) |
| } |
| - if _, found, err := LoadWake(ctx, s, pfx, m); err == nil || found { |
| + if _, found, err := LoadWake(ctx, s, pfx, m, nil); err == nil || found { |
| t.Fatalf("found = %v, err = %v; a missing wake object must be an error", found, err) |
| } |
| } |