amadan

carlos / platform Public

Preserve maintenance projection across provision record writes

Paul Campbell pushed by paul@keymail.dev b0c51dbb0da11cd610b71df7d347c855069bf91c parent 2abaf89
4 files changed, +369 −18
  • internal/provision/maintenance.go +94 −0
  • internal/provision/maintenance_test.go +233 −0
  • internal/provision/provision.go +38 −14
  • internal/provision/reconcile.go +4 −4
diff --git a/internal/provision/maintenance.go b/internal/provision/maintenance.go
new file mode 100644
index 000000000..dc8b6aa90
--- /dev/null
+++ b/internal/provision/maintenance.go
@@ -0,0 +1,94 @@
+package provision
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "io"
+ "strings"
+
+ "github.com/carlosframework/platform/internal/maintenanceguard"
+)
+
+var ErrMaintenanceProjection = errors.New("provision: invalid maintenance projection or protected fields")
+
+// MaintenanceProjection retains a present projection, including invalid input.
+// Its value receiver validates on marshal, so an ignored decode error cannot
+// turn a present invalid projection into an absent one. An empty slice is absent.
+type MaintenanceProjection []byte
+
+func (p *MaintenanceProjection) UnmarshalJSON(raw []byte) error {
+ *p = append((*p)[:0], raw...)
+ _, err := maintenanceguard.DecodeProjection(raw)
+ if err != nil {
+ return ErrMaintenanceProjection
+ }
+ return nil
+}
+func (p MaintenanceProjection) MarshalJSON() ([]byte, error) {
+ if _, err := maintenanceguard.DecodeProjection(p); err != nil {
+ return nil, ErrMaintenanceProjection
+ }
+ return append([]byte(nil), p...), nil
+}
+
+// DecodeRecord is the bucket-record parser. API response views may use ordinary
+// JSON decoding; they do not authorize writes. Keep this off Record's method
+// set so embedding Record cannot swallow a wrapper's fields or Gate shadow.
+func DecodeRecord(raw []byte) (Record, error) {
+ invalid := Record{Maintenance: MaintenanceProjection("null")}
+ dec := json.NewDecoder(bytes.NewReader(raw))
+ first, e := dec.Token()
+ if e != nil || first != json.Delim('{') {
+ return invalid, ErrMaintenanceProjection
+ }
+ seen := false
+ for dec.More() {
+ key, e := dec.Token()
+ if e != nil {
+ return invalid, ErrMaintenanceProjection
+ }
+ name, ok := key.(string)
+ if !ok {
+ return invalid, ErrMaintenanceProjection
+ }
+ if strings.EqualFold(name, "maintenance") {
+ if name != "maintenance" || seen {
+ return invalid, ErrMaintenanceProjection
+ }
+ seen = true
+ }
+ var value json.RawMessage
+ if dec.Decode(&value) != nil {
+ return invalid, ErrMaintenanceProjection
+ }
+ }
+ if end, e := dec.Token(); e != nil || end != json.Delim('}') {
+ return invalid, ErrMaintenanceProjection
+ }
+ if _, e = dec.Token(); e != io.EOF {
+ return invalid, ErrMaintenanceProjection
+ }
+ var rec Record
+ if json.Unmarshal(raw, &rec) != nil {
+ return invalid, ErrMaintenanceProjection
+ }
+ return rec, nil
+}
+
+// Equality preserves the projection, not permission to change a fenced record.
+// Durable guard and lifecycle checks belong to the maintenance integration.
+func protectedFieldsEqual(before, after Record) bool {
+ if before.Account != after.Account || before.App != after.App || before.Host != after.Host || !before.CreatedAt.Equal(after.CreatedAt) {
+ return false
+ }
+ if len(before.Maintenance) == 0 || len(after.Maintenance) == 0 {
+ return len(before.Maintenance) == len(after.Maintenance)
+ }
+ a, e := maintenanceguard.DecodeProjection(before.Maintenance)
+ if e != nil {
+ return false
+ }
+ b, e := maintenanceguard.DecodeProjection(after.Maintenance)
+ return e == nil && a == b
+}
diff --git a/internal/provision/maintenance_test.go b/internal/provision/maintenance_test.go
new file mode 100644
index 000000000..177fa8ffa
--- /dev/null
+++ b/internal/provision/maintenance_test.go
@@ -0,0 +1,233 @@
+package provision
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/maintenanceguard"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+func testProjection(t *testing.T, generation int64) MaintenanceProjection {
+ t.Helper()
+ b, e := maintenanceguard.EncodeProjection(maintenanceguard.Projection{Version: 1, Generation: generation, GuardDigest: strings.Repeat("a", 64)})
+ if e != nil {
+ t.Fatal(e)
+ }
+ return b
+}
+func TestMaintenanceProjectionParsing(t *testing.T) {
+ valid := string(testProjection(t, 1))
+ for _, value := range []string{"null", "{}", "[]", `"private"`, strings.Replace(valid, `"version":1`, `"version":2`, 1), strings.Replace(valid, `"generation":1`, `"generation":1.0`, 1), strings.Replace(valid, `"generation":1`, `"generation":0`, 1), strings.Replace(valid, `"version":1`, `"version":1,"version":1`, 1), strings.Replace(valid, `"version":1`, `"Version":1`, 1), strings.Replace(valid, `"version":1`, `"version":1,"extra":true`, 1)} {
+ t.Run(value, func(t *testing.T) {
+ raw := []byte(`{"maintenance":` + value + `}`)
+ r, e := DecodeRecord(raw)
+ if e != ErrMaintenanceProjection {
+ t.Fatal("accepted invalid")
+ }
+ if _, e = json.Marshal(r); e == nil {
+ t.Fatal("invalid disappeared")
+ }
+ var direct Record
+ if json.Unmarshal(raw, &direct) == nil {
+ t.Fatal("field accepted invalid")
+ }
+ if _, e = json.Marshal(direct); e == nil {
+ t.Fatal("ignored field error disappeared")
+ }
+ })
+ }
+ for _, raw := range []string{`{"maintenance":` + valid + `,"maintenance":` + valid + `}`, `{"Maintenance":` + valid + `}`, `{"maintenance":` + valid + `,"MAINTENANCE":null}`, `null`, `{} {}`} {
+ if _, e := DecodeRecord([]byte(raw)); e == nil {
+ t.Fatal("accepted outer ambiguity")
+ }
+ }
+ r, e := DecodeRecord([]byte(`{"maintenance":` + valid + `,"future_record_field":true}`))
+ if e != nil {
+ t.Fatal(e)
+ }
+ if _, e = json.Marshal(r); e != nil {
+ t.Fatal(e)
+ }
+ legacy := Record{Host: "legacy"}
+ b, e := json.Marshal(legacy)
+ if e != nil || bytes.Contains(b, []byte("maintenance")) {
+ t.Fatal("legacy changed", e)
+ }
+}
+func TestMaintenanceDoesNotPromoteJSONMethods(t *testing.T) {
+ type view struct {
+ Record
+ HiddenGate string `json:"gate,omitempty"`
+ Extra string `json:"extra"`
+ }
+ in := view{Record: Record{Gate: "private hash", Maintenance: testProjection(t, 1)}, Extra: "joined"}
+ b, e := json.Marshal(in)
+ if e != nil || bytes.Contains(b, []byte("private hash")) {
+ t.Fatal("gate leaked", e)
+ }
+ var out view
+ if json.Unmarshal(b, &out) != nil || out.Extra != "joined" || len(out.Maintenance) == 0 {
+ t.Fatal("embedded fields swallowed")
+ }
+}
+func TestOrdinaryWritersPreserveProjection(t *testing.T) {
+ s, sqid := setup(t)
+ rec := acmeRecord(sqid)
+ rec.Maintenance = testProjection(t, 1)
+ if CreateRecord(ctx, s, rec) != ErrMaintenanceProjection {
+ t.Fatal("ordinary create installed projection")
+ }
+ key := RecordKey(rec.Account, rec.App, rec.Host)
+ raw, e := json.Marshal(rec)
+ if e != nil {
+ t.Fatal(e)
+ }
+ if _, e = s.Put(ctx, key, raw, ""); e != nil {
+ t.Fatal(e)
+ }
+ for _, mut := range []func(*Record){func(r *Record) { r.Maintenance = nil }, func(r *Record) { r.Maintenance = testProjection(t, 2) }, func(r *Record) { r.CreatedAt = r.CreatedAt.Add(time.Second) }, func(r *Record) { r.Account = "different" }, func(r *Record) { r.App = "different" }, func(r *Record) { r.Host = "different.example" }} {
+ before, etag, e := GetRecordETag(ctx, s, rec.Account, rec.App, rec.Host)
+ if e != nil {
+ t.Fatal(e)
+ }
+ mut(before)
+ if UpdateRecord(ctx, s, *before, etag) == nil {
+ t.Fatal("whole writer changed protected fields")
+ }
+ if _, e = SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, func(r *Record) error { mut(r); return nil }); e == nil {
+ t.Fatal("callback changed protected fields")
+ }
+ }
+ current, etag, e := GetRecordETag(ctx, s, rec.Account, rec.App, rec.Host)
+ if e != nil {
+ t.Fatal(e)
+ }
+ current.Maintenance = MaintenanceProjection(strings.Replace(string(current.Maintenance), ":", ": ", -1))
+ current.CreatedAt = current.CreatedAt.In(time.FixedZone("same instant", 3600))
+ current.Channel = "new"
+ if e = UpdateRecord(ctx, s, *current, etag); e != nil {
+ t.Fatal("semantic equality refused", e)
+ }
+ if e = SetRecordRegion(ctx, s, rec.Account, rec.App, rec.Host, "uk"); e != nil {
+ t.Fatal(e)
+ }
+ got, e := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
+ if e != nil || !protectedFieldsEqual(rec, *got) || got.Region != "uk" {
+ t.Fatal("projection lost", e)
+ }
+ absent := acmeRecord(sqid)
+ absent.Host = "other.go.tito.io"
+ if e = CreateRecord(ctx, s, absent); e != nil {
+ t.Fatal(e)
+ }
+ if _, e = SetRecordMove(ctx, s, absent.Account, absent.App, absent.Host, func(r *Record) error { r.Maintenance = testProjection(t, 1); return nil }); e != ErrMaintenanceProjection {
+ t.Fatal("installed into absent record", e)
+ }
+}
+
+type projectionRaceStore struct {
+ release.Store
+ beforePut func()
+}
+
+func (s *projectionRaceStore) Put(c context.Context, k string, b []byte, tag string) (string, error) {
+ if s.beforePut != nil {
+ f := s.beforePut
+ s.beforePut = nil
+ f()
+ }
+ return s.Store.Put(c, k, b, tag)
+}
+func TestProjectionCASRaces(t *testing.T) {
+ for _, retry := range []bool{false, true} {
+ t.Run(map[bool]string{false: "whole", true: "retry"}[retry], func(t *testing.T) {
+ base, sqid := setup(t)
+ rec := acmeRecord(sqid)
+ if e := CreateRecord(ctx, base, rec); e != nil {
+ t.Fatal(e)
+ }
+ old, tag, e := GetRecordETag(ctx, base, rec.Account, rec.App, rec.Host)
+ if e != nil {
+ t.Fatal(e)
+ }
+ wrapped := &projectionRaceStore{Store: base, beforePut: func() {
+ rec.Maintenance = testProjection(t, 1)
+ b, e := json.Marshal(rec)
+ if e != nil {
+ t.Fatal(e)
+ }
+ putReplace(t, base, RecordKey(rec.Account, rec.App, rec.Host), b)
+ }}
+ if retry {
+ e = SetRecordRegion(ctx, wrapped, rec.Account, rec.App, rec.Host, "uk")
+ if e != nil {
+ t.Fatal(e)
+ }
+ } else {
+ old.Channel = "new"
+ e = UpdateRecord(ctx, wrapped, *old, tag)
+ if !errors.Is(e, release.ErrCASConflict) {
+ t.Fatal("stale write won", e)
+ }
+ }
+ got, e := GetRecord(ctx, base, rec.Account, rec.App, rec.Host)
+ if e != nil || len(got.Maintenance) == 0 {
+ t.Fatal("concurrent projection stripped", e)
+ }
+ if !retry && got.Channel != rec.Channel {
+ t.Fatal("failed write applied")
+ }
+ })
+ }
+}
+func TestMalformedProjectionReconcileAndLists(t *testing.T) {
+ s, sqid := setup(t)
+ rec := acmeRecord(sqid)
+ if e := CreateRecord(ctx, s, rec); e != nil {
+ t.Fatal(e)
+ }
+ reg := regWith(t)
+ cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}
+ if e := Reconcile(ctx, s, reg, cfg); e != nil {
+ t.Fatal(e)
+ }
+ for _, host := range []string{rec.Host, "new.go.tito.io"} {
+ rec.Host = host
+ if host == "new.go.tito.io" {
+ if e := CreateRecord(ctx, s, rec); e != nil {
+ t.Fatal(e)
+ }
+ }
+ b, e := json.Marshal(rec)
+ if e != nil {
+ t.Fatal(e)
+ }
+ b = append(b[:len(b)-1], []byte(`,"maintenance":null}`)...)
+ putReplace(t, s, RecordKey(sqid, "tito", host), b)
+ if _, _, e = GetRecordETag(ctx, s, sqid, "tito", host); e == nil {
+ t.Fatal("get accepted invalid")
+ }
+ if e = SetRecordRegion(ctx, s, sqid, "tito", host, "uk"); e == nil {
+ t.Fatal("writer accepted invalid")
+ }
+ }
+ if e := Reconcile(ctx, s, reg, cfg); e != nil {
+ t.Fatal(e)
+ }
+ mustGet(t, reg, "acme.go.tito.io")
+ if r, e := reg.Get("new.go.tito.io"); e != nil || r != nil {
+ t.Fatal("malformed created route")
+ }
+ if records, _, e := ListRecords(ctx, s, sqid, "tito", 10); e != nil || len(records) != 0 {
+ t.Fatal("list semantics changed")
+ }
+ if records, e := ListAllRecords(ctx, s); e != nil || len(records) != 0 {
+ t.Fatal("all-list semantics changed")
+ }
+}
diff --git a/internal/provision/provision.go b/internal/provision/provision.go
index 887cb538a..c5da151c6 100644
--- a/internal/provision/provision.go
+++ b/internal/provision/provision.go
@@ -34,16 +34,19 @@ import (
// Account vs the key's segments); GetRecord and ListRecords enforce that, a
// record that lies about identity is never guessed at (design §3.2).
type Record struct {
- Schema int `json:"schema"` // 1
- Host string `json:"host"`
- Slug string `json:"slug"`
- App string `json:"app"`
- Account string `json:"account"`
- Pool string `json:"pool,omitempty"`
- Region string `json:"region,omitempty"`
- Channel string `json:"channel"`
- CreatedAt time.Time `json:"created_at"`
- CreatedBy string `json:"created_by,omitempty"`
+ // Maintenance is preserved by ordinary writers; only a future guarded
+ // maintenance transition may install, replace, or clear it.
+ Maintenance MaintenanceProjection `json:"maintenance,omitempty"`
+ Schema int `json:"schema"` // 1
+ Host string `json:"host"`
+ Slug string `json:"slug"`
+ App string `json:"app"`
+ Account string `json:"account"`
+ Pool string `json:"pool,omitempty"`
+ Region string `json:"region,omitempty"`
+ Channel string `json:"channel"`
+ CreatedAt time.Time `json:"created_at"`
+ CreatedBy string `json:"created_by,omitempty"`
// Move is an in-flight residence move (issue #331, instance-move spec):
// present only between move-prepare and the cut's terminal flip, absent
@@ -625,6 +628,9 @@ func PutManifestIfMatch(ctx context.Context, s release.Store, acct, app string,
// against a second signup for the same host comes back wrapping
// release.ErrCASConflict, never a silent overwrite of the winner's record.
func CreateRecord(ctx context.Context, s release.Store, rec Record) error {
+ if len(rec.Maintenance) != 0 {
+ return ErrMaintenanceProjection
+ }
body, err := json.Marshal(rec)
if err != nil {
return err
@@ -673,13 +679,21 @@ func updateRecord(ctx context.Context, s release.Store, acct, app, host string,
if err != nil {
return nil, err
}
- var rec Record
- if err := json.Unmarshal(raw, &rec); err != nil {
+ rec, err := DecodeRecord(raw)
+ if err != nil {
return nil, fmt.Errorf("provision: record %s/%s/%s: %w", acct, app, host, err)
}
+ if rec.Account != acct || rec.App != app || rec.Host != host {
+ return nil, ErrMaintenanceProjection
+ }
+ before := rec
+ before.Maintenance = append(MaintenanceProjection(nil), rec.Maintenance...)
if err := mut(&rec); err != nil {
return nil, err
}
+ if !protectedFieldsEqual(before, rec) {
+ return nil, ErrMaintenanceProjection
+ }
body, err := json.Marshal(rec)
if err != nil {
return nil, err
@@ -881,6 +895,16 @@ func UpdateRecord(ctx context.Context, s release.Store, rec Record, etag string)
return fmt.Errorf("provision: update %s/%s/%s: no etag — an update must be guarded by the read it was decided from",
rec.Account, rec.App, rec.Host)
}
+ before, currentETag, err := GetRecordETag(ctx, s, rec.Account, rec.App, rec.Host)
+ if err != nil {
+ return err
+ }
+ if currentETag != etag {
+ return release.ErrCASConflict
+ }
+ if !protectedFieldsEqual(*before, rec) {
+ return ErrMaintenanceProjection
+ }
body, err := json.Marshal(rec)
if err != nil {
return err
@@ -907,8 +931,8 @@ func GetRecordETag(ctx context.Context, s release.Store, acct, app, host string)
if err != nil {
return nil, "", err
}
- var rec Record
- if err := json.Unmarshal(raw, &rec); err != nil {
+ rec, err := DecodeRecord(raw)
+ if err != nil {
return nil, "", fmt.Errorf("provision: record %s/%s/%s: %w", acct, app, host, err)
}
if rec.Account != acct || rec.App != app || rec.Host != host {
diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go
index 610088ef8..0d2144dee 100644
--- a/internal/provision/reconcile.go
+++ b/internal/provision/reconcile.go
@@ -619,8 +619,8 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
readFailed = true
continue
}
- var rec Record
- if err := json.Unmarshal(raw, &rec); err != nil {
+ rec, err := DecodeRecord(raw)
+ if err != nil {
log.Printf("provision: skipping record %s: malformed: %v", key, err)
protect[host] = true
continue
@@ -1126,8 +1126,8 @@ func ListAllRecords(ctx context.Context, s release.Store) ([]Record, error) {
if err != nil {
return nil, fmt.Errorf("provision: read record %s: %w", key, err)
}
- var rec Record
- if err := json.Unmarshal(raw, &rec); err != nil {
+ rec, err := DecodeRecord(raw)
+ if err != nil {
log.Printf("provision: skipping record %s: malformed: %v", key, err)
continue
}