| 1 | // Package hostfiles publishes an instance's files directory, and the small |
| 2 | // platform-managed items that travel with it, to object storage in an order |
| 3 | // that survives late and duplicate uploads (region HA spec 1 §5.6). |
| 4 | // |
| 5 | // Nothing is overwritten: each change is a new immutable object, and one |
| 6 | // manifest, replaced by compare-and-swap, says which object is current. |
| 7 | package hostfiles |
| 8 | |
| 9 | import ( |
| 10 | "context" |
| 11 | "encoding/json" |
| 12 | "errors" |
| 13 | "fmt" |
| 14 | "os" |
| 15 | "path" |
| 16 | "strings" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/release" |
| 19 | ) |
| 20 | |
| 21 | // MaxFileBytes is the largest file published as one object. release.Store |
| 22 | // takes a whole body in memory, so a larger file is recorded as skipped |
| 23 | // rather than read. Plan 1b lifts this with a streaming put. |
| 24 | const MaxFileBytes = 64 << 20 |
| 25 | |
| 26 | // Entry is one path's state. Object is origin-qualified: the object's full |
| 27 | // key from the bucket root, "<origin>objects/<seq>-<sha256>", where origin |
| 28 | // is the files prefix of the generation that wrote it. A manifest copied |
| 29 | // into a later generation keeps every Object as it was, so the copy names |
| 30 | // the same immutable objects and nothing is uploaded again (claim L13). |
| 31 | // Readers resolve Object exactly as written, never against their own |
| 32 | // prefix, and only within the origins their caller admits. |
| 33 | type Entry struct { |
| 34 | Object string `json:"object,omitempty"` |
| 35 | SHA256 string `json:"sha256,omitempty"` |
| 36 | Size int64 `json:"size,omitempty"` |
| 37 | Mode uint32 `json:"mode,omitempty"` |
| 38 | Seq int64 `json:"seq"` |
| 39 | Deleted bool `json:"deleted,omitempty"` |
| 40 | Torn bool `json:"torn,omitempty"` |
| 41 | Skipped string `json:"skipped,omitempty"` |
| 42 | } |
| 43 | |
| 44 | // ManifestVersion is the manifest format this binary reads and writes. |
| 45 | // |
| 46 | // Version 2 made Entry.Object origin-qualified (a full key) where version |
| 47 | // 1 held a key relative to the manifest's prefix. That changes what an |
| 48 | // existing field means; it adds nothing a reader could ignore. A version 1 |
| 49 | // reader would join a qualified key to its own prefix and look in the |
| 50 | // wrong place, and a version 1 manifest copied to another generation |
| 51 | // would resolve against the copy's prefix. So the version is bumped and |
| 52 | // every other version is refused. No version 1 manifest was written in |
| 53 | // production (nothing called the publisher), so there is nothing to |
| 54 | // migrate. |
| 55 | const ManifestVersion = 2 |
| 56 | |
| 57 | type Manifest struct { |
| 58 | V int `json:"v"` |
| 59 | Seq int64 `json:"seq"` |
| 60 | Entries map[string]Entry `json:"entries"` |
| 61 | Platform map[string]Entry `json:"platform"` |
| 62 | // Origins is the admitted origin set the manifest carries forward |
| 63 | // (moves Task 6 fix round 2, platform-1e's ruling): the generation |
| 64 | // files prefixes whose objects its entries may name, sorted. A |
| 65 | // prepared manifest seeds it with its own prefix; each admission's |
| 66 | // establishment sets the new manifest's to the previous manifest's set |
| 67 | // plus the generations admitted in the then-current chain plus its |
| 68 | // own prefix, so a pruned ancestor stays proven by it and a generation |
| 69 | // never admitted cannot enter it. A manifest without the field (the |
| 70 | // format before this) carries none, and a reader falls back to the |
| 71 | // admitted chain alone. Additive: every other field is as before. |
| 72 | Origins []string `json:"origins,omitempty"` |
| 73 | } |
| 74 | |
| 75 | func ManifestKey(prefix string) string { return prefix + "manifest.json" } |
| 76 | |
| 77 | // objectKey is an object's key relative to the origin that wrote it. |
| 78 | func objectKey(seq int64, sha string) string { return fmt.Sprintf("objects/%016d-%s", seq, sha) } |
| 79 | |
| 80 | // ErrOrigin marks an entry whose object lies outside the origins its |
| 81 | // reader admits, or an origin that is not a well-formed prefix. |
| 82 | var ErrOrigin = errors.New("hostfiles: object origin not admitted") |
| 83 | |
| 84 | // checkPrefix accepts a generation prefix: a clean, relative bucket key |
| 85 | // ending in "/". |
| 86 | func checkPrefix(prefix string) error { |
| 87 | dir, ok := strings.CutSuffix(prefix, "/") |
| 88 | if !ok { |
| 89 | return fmt.Errorf("hostfiles: prefix %q must end in /", prefix) |
| 90 | } |
| 91 | if _, err := CleanPath(dir); err != nil { |
| 92 | return fmt.Errorf("hostfiles: prefix %q is not a clean bucket key", prefix) |
| 93 | } |
| 94 | return nil |
| 95 | } |
| 96 | |
| 97 | // admitted resolves a caller's origins: nil or empty means the manifest's |
| 98 | // own prefix alone. Every origin must be a well-formed prefix. |
| 99 | func admitted(prefix string, origins []string) ([]string, error) { |
| 100 | if len(origins) == 0 { |
| 101 | origins = []string{prefix} |
| 102 | } |
| 103 | for _, o := range origins { |
| 104 | if err := checkPrefix(o); err != nil { |
| 105 | return nil, fmt.Errorf("%w: %v", ErrOrigin, err) |
| 106 | } |
| 107 | } |
| 108 | return origins, nil |
| 109 | } |
| 110 | |
| 111 | // CleanPath accepts only a relative, already-clean, slash-separated path |
| 112 | // that stays inside its root. It never repairs a path: a manifest is |
| 113 | // written by this package, so anything else is damage or an attack. |
| 114 | func CleanPath(p string) (string, error) { |
| 115 | if p == "" || strings.ContainsRune(p, 0) || strings.HasPrefix(p, "/") { |
| 116 | return "", fmt.Errorf("hostfiles: path %q is not a relative path", p) |
| 117 | } |
| 118 | c := path.Clean(p) |
| 119 | if c != p || c == "." || c == ".." || strings.HasPrefix(c, "../") { |
| 120 | return "", fmt.Errorf("hostfiles: path %q is not clean or leaves its root", p) |
| 121 | } |
| 122 | return c, nil |
| 123 | } |
| 124 | |
| 125 | // Load reads the manifest. The bool is false when there is none yet. |
| 126 | func Load(ctx context.Context, s release.Store, prefix string) (Manifest, bool, error) { |
| 127 | m, _, found, err := load(ctx, s, prefix) |
| 128 | return m, found, err |
| 129 | } |
| 130 | |
| 131 | func load(ctx context.Context, s release.Store, prefix string) (Manifest, string, bool, error) { |
| 132 | empty := Manifest{V: ManifestVersion, Entries: map[string]Entry{}, Platform: map[string]Entry{}} |
| 133 | raw, etag, err := s.Get(ctx, ManifestKey(prefix)) |
| 134 | if errors.Is(err, os.ErrNotExist) { |
| 135 | return empty, "", false, nil |
| 136 | } |
| 137 | if err != nil { |
| 138 | return Manifest{}, "", false, err |
| 139 | } |
| 140 | m, err := decode(prefix, raw) |
| 141 | if err != nil { |
| 142 | return Manifest{}, "", false, err |
| 143 | } |
| 144 | return m, etag, true, nil |
| 145 | } |
| 146 | |
| 147 | // decode is a manifest from its exact bytes. |
| 148 | func decode(prefix string, raw []byte) (Manifest, error) { |
| 149 | var m Manifest |
| 150 | if err := json.Unmarshal(raw, &m); err != nil { |
| 151 | return Manifest{}, fmt.Errorf("hostfiles: manifest %s: %w", prefix, err) |
| 152 | } |
| 153 | if m.V != ManifestVersion { |
| 154 | return Manifest{}, fmt.Errorf("hostfiles: manifest %s is version %d; this binary reads %d", prefix, m.V, ManifestVersion) |
| 155 | } |
| 156 | if m.Entries == nil { |
| 157 | m.Entries = map[string]Entry{} |
| 158 | } |
| 159 | if m.Platform == nil { |
| 160 | m.Platform = map[string]Entry{} |
| 161 | } |
| 162 | return m, nil |
| 163 | } |
| 164 | |
amadan