| 1 | // The scan sidecar (spec §5): apps/<acct>/<app>/scan.json, the coverage |
| 2 | // rollup, and the object an audit answer is quoted from. |
| 3 | // |
| 4 | // It is a SEPARATE OBJECT from store.json on purpose. The sweep writes once |
| 5 | // per page over minutes; the create, grant and rotate doors write store.json |
| 6 | // too. Sharing one object would put a page-write in a compare-and-swap race |
| 7 | // with every repair walk, and losing that race is the normal case rather than |
| 8 | // the exception. With its own object and a single writer, the whole CAS story |
| 9 | // shrinks to "another sweep is ahead of me, stand down". |
| 10 | package objstore |
| 11 | |
| 12 | import ( |
| 13 | "context" |
| 14 | "encoding/json" |
| 15 | "errors" |
| 16 | "fmt" |
| 17 | "os" |
| 18 | "time" |
| 19 | |
| 20 | "github.com/carlosframework/platform/internal/release" |
| 21 | ) |
| 22 | |
| 23 | const SidecarSchema = 1 |
| 24 | |
| 25 | // ErrSidecarMismatch is a sidecar whose body disagrees with the path it was |
| 26 | // read from. |
| 27 | var ErrSidecarMismatch = errors.New("objstore: scan sidecar names another app") |
| 28 | |
| 29 | func ScanKey(acct, app string) string { return release.AppPrefix(acct, app) + "scan.json" } |
| 30 | |
| 31 | // Sweep is one reconciliation pass over one bucket. |
| 32 | // |
| 33 | // Scanned and Unscanned are NOT complements. Skipped, Denied and Failed are |
| 34 | // recorded verdicts that are not coverage — an object too large to stream was |
| 35 | // never judged, and calling it clean would be a lie an auditor is entitled to |
| 36 | // catch. That is why the audit sentence names two numbers. |
| 37 | type Sweep struct { |
| 38 | StartedAt time.Time `json:"started_at"` |
| 39 | At time.Time `json:"at"` |
| 40 | Engine string `json:"engine"` |
| 41 | // Signature is the engine's signature-database version, verbatim. A clean |
| 42 | // verdict from three-month-old signatures should not be indistinguishable |
| 43 | // from a fresh one. |
| 44 | Signature string `json:"signature,omitempty"` |
| 45 | Objects int64 `json:"objects"` |
| 46 | Scanned int64 `json:"scanned"` |
| 47 | Infected int64 `json:"infected"` |
| 48 | Skipped int64 `json:"skipped"` |
| 49 | Denied int64 `json:"denied"` |
| 50 | Failed int64 `json:"failed"` |
| 51 | Unscanned int64 `json:"unscanned"` |
| 52 | // Fresh and Reused split Scanned by WHEN the verdict was reached. A clean |
| 53 | // tag is reused for as long as the object's ETag matches, so a completed |
| 54 | // sweep can be built almost entirely from judgements made months ago, |
| 55 | // against a signature database that is not the one named above. Without |
| 56 | // this split, Signature reads as a claim about every object in the rollup, |
| 57 | // and it is not one. |
| 58 | Fresh int64 `json:"fresh"` |
| 59 | Reused int64 `json:"reused"` |
| 60 | Bytes int64 `json:"bytes"` |
| 61 | // After is the last key of the last written page — the resume point. |
| 62 | // A key, not a ListObjectsV2 continuation token: a token's lifetime across |
| 63 | // processes is unspecified and, on Ceph, unmeasured, while a key is |
| 64 | // portable to any S3-compatible store. |
| 65 | After string `json:"after,omitempty"` |
| 66 | } |
| 67 | |
| 68 | // ScanSidecar holds the in-flight sweep and the last completed one. |
| 69 | // |
| 70 | // TWO SWEEPS, NOT ONE. A single field would have to mean both "in progress" |
| 71 | // and "the answer", and the tick after a sweep completed would overwrite the |
| 72 | // completed counts with page-one partials — destroying the compliance artifact |
| 73 | // on every cycle. Current accumulates; on its last page it moves to |
| 74 | // LastComplete, and LastComplete is the only thing an audit answer quotes. |
| 75 | type ScanSidecar struct { |
| 76 | Schema int `json:"schema"` |
| 77 | Account string `json:"account"` |
| 78 | App string `json:"app"` |
| 79 | // NotApplicable is a store whose member attested client-side encryption. |
| 80 | // A scanner cannot read ciphertext, so no sweep runs and no coverage is |
| 81 | // claimed — reporting "all clean" for such a store would be worse than |
| 82 | // reporting nothing. |
| 83 | NotApplicable bool `json:"not_applicable,omitempty"` |
| 84 | Current *Sweep `json:"current,omitempty"` |
| 85 | LastComplete *Sweep `json:"last_complete,omitempty"` |
| 86 | } |
| 87 | |
| 88 | // GetSidecar reads the sidecar. Absent is (nil, "", nil) — no sidecar is a |
| 89 | // state, not an error, the same idiom Get uses for an absent record. |
| 90 | func GetSidecar(ctx context.Context, s release.Store, acct, app string) (*ScanSidecar, string, error) { |
| 91 | if !release.ValidAccount(acct) { |
| 92 | return nil, "", fmt.Errorf("objstore: invalid account %q", acct) |
| 93 | } |
| 94 | if !release.ValidApp(app) { |
| 95 | return nil, "", fmt.Errorf("objstore: invalid app %q", app) |
| 96 | } |
| 97 | raw, etag, err := s.Get(ctx, ScanKey(acct, app)) |
| 98 | if err != nil { |
| 99 | if errors.Is(err, os.ErrNotExist) { |
| 100 | return nil, "", nil |
| 101 | } |
| 102 | return nil, "", err |
| 103 | } |
| 104 | var sc ScanSidecar |
| 105 | if err := json.Unmarshal(raw, &sc); err != nil { |
| 106 | return nil, "", fmt.Errorf("objstore: %s: %w", ScanKey(acct, app), err) |
| 107 | } |
| 108 | // An unknown schema is refused rather than guessed at — the rule |
| 109 | // release.ReadAppTombstone already applies to its own shape. |
| 110 | if sc.Schema != SidecarSchema { |
| 111 | return nil, "", fmt.Errorf("objstore: %s: schema %d is not %d", |
| 112 | ScanKey(acct, app), sc.Schema, SidecarSchema) |
| 113 | } |
| 114 | // `accounts migrate` copies every key under an app's prefix, so a sidecar |
| 115 | // can arrive under an account it was never written for, carrying rollups |
| 116 | // for a bucket named after the old one. Refusing beats reporting another |
| 117 | // account's coverage as this one's. |
| 118 | if sc.Account != acct || sc.App != app { |
| 119 | return nil, "", fmt.Errorf("%w: body says %s/%s, path says %s/%s", |
| 120 | ErrSidecarMismatch, sc.Account, sc.App, acct, app) |
| 121 | } |
| 122 | return &sc, etag, nil |
| 123 | } |
| 124 | |
| 125 | // PutSidecar writes with the caller's CAS position; etag "" is create-only. |
| 126 | // It stamps Schema, Account and App the way Put stamps Schema and App, so a |
| 127 | // caller cannot write a body that disagrees with its own path. |
| 128 | func claimRawPutSidecar(ctx context.Context, s release.Store, acct, app string, sc *ScanSidecar, etag string) (string, error) { |
| 129 | sc.Schema = SidecarSchema |
| 130 | sc.Account = acct |
| 131 | sc.App = app |
| 132 | body, err := json.MarshalIndent(sc, "", " ") |
| 133 | if err != nil { |
| 134 | return "", err |
| 135 | } |
| 136 | return s.Put(ctx, ScanKey(acct, app), append(body, '\n'), etag) |
| 137 | } |
| 138 | func PutSidecar(ctx context.Context, s release.Store, acct, app string, sc *ScanSidecar, etag string) (claimResult0 string, claimResult1 error) { |
| 139 | claimResult1 = release.WithAppMutation(ctx, s, acct, app, "objstore.PutSidecar", func(claimContext context.Context) error { |
| 140 | claimResult0, claimResult1 = claimRawPutSidecar(claimContext, s, acct, app, sc, etag) |
| 141 | return claimResult1 |
| 142 | }) |
| 143 | return |
| 144 | } |
| 145 | |
amadan