| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "net/http" |
| 7 | "testing" |
| 8 | "time" |
| 9 | |
| 10 | "github.com/carlosframework/platform/internal/hostclaim" |
| 11 | "github.com/carlosframework/platform/internal/hostops" |
| 12 | "github.com/carlosframework/platform/internal/provision" |
| 13 | "github.com/carlosframework/platform/internal/release" |
| 14 | ) |
| 15 | |
| 16 | // J17 rev2: the typed door binds the fresh observation to the move and record |
| 17 | // its operator gate read. A store wrapper runs a replacement between the two |
| 18 | // reads, on the real door. |
| 19 | |
| 20 | // j17RaceStore runs hook once, just before the door's first read of the |
| 21 | // host claim (the fresh ObserveConsoleMove, after the gate's record read). |
| 22 | type j17RaceStore struct { |
| 23 | release.Store |
| 24 | key string |
| 25 | hook func() |
| 26 | } |
| 27 | |
| 28 | func (s *j17RaceStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 29 | if key == s.key && s.hook != nil { |
| 30 | h := s.hook |
| 31 | s.hook = nil |
| 32 | h() |
| 33 | } |
| 34 | return s.Store.Get(ctx, key) |
| 35 | } |
| 36 | |
| 37 | func j17Arm(fx *consoleFixture, hook func()) { |
| 38 | fx.srv.cfg.Releases = &j17RaceStore{Store: fx.srv.cfg.Releases, key: "control/hosts/acme.go.tito.io.json", hook: hook} |
| 39 | } |
| 40 | |
| 41 | // j17Replace swaps the claim's op, the record's move and the registration to |
| 42 | // a successor move id, as a finished move and a new request would. |
| 43 | func j17Replace(t *testing.T, fx *consoleFixture, oldID string) { |
| 44 | t.Helper() |
| 45 | ctx := context.Background() |
| 46 | id := "mv_abcdefghijklmnopqrstuvwxyz" |
| 47 | if id == oldID { |
| 48 | id = "mv_zyxwvutsrqponmlkjihgfedcba" |
| 49 | } |
| 50 | obs, err := hostclaim.Read(ctx, fx.rel, "acme.go.tito.io") |
| 51 | if err != nil { |
| 52 | t.Fatal(err) |
| 53 | } |
| 54 | if _, err := hostclaim.Update(ctx, fx.rel, obs, func(c *hostclaim.Claim) error { |
| 55 | c.Op.ID, c.Op.Intent["admission_id"] = id, id |
| 56 | return nil |
| 57 | }); err != nil { |
| 58 | t.Fatal(err) |
| 59 | } |
| 60 | j17SetRecord(t, fx, func(r *provision.Record) { r.Move.ID = id }) |
| 61 | a, err := hostops.ReadAdmission(ctx, fx.rel, "bab", "titogo", claimMoveAppID) |
| 62 | if err != nil { |
| 63 | t.Fatal(err) |
| 64 | } |
| 65 | a.Value.Registrations[id] = a.Value.Registrations[oldID] |
| 66 | b, _ := json.Marshal(a.Value) |
| 67 | if _, err := fx.rel.Put(ctx, a.Key, b, a.ETag); err != nil { |
| 68 | t.Fatal(err) |
| 69 | } |
| 70 | } |
| 71 | |
| 72 | func j17RecordBytes(t *testing.T, fx *consoleFixture) string { |
| 73 | t.Helper() |
| 74 | b, _, err := fx.rel.Get(context.Background(), provision.RecordKey("bab", "titogo", "acme.go.tito.io")) |
| 75 | if err != nil { |
| 76 | t.Fatal(err) |
| 77 | } |
| 78 | return string(b) |
| 79 | } |
| 80 | |
| 81 | func j17Serial(t *testing.T, fx *consoleFixture) int64 { |
| 82 | t.Helper() |
| 83 | d, _, err := release.ReadSerial(context.Background(), fx.rel) |
| 84 | if err != nil { |
| 85 | t.Fatal(err) |
| 86 | } |
| 87 | return d.Serial |
| 88 | } |
| 89 | |
| 90 | // j17Raced fires the request with the hook between the gate and the |
| 91 | // observation and requires the identity 409 with nothing written. |
| 92 | func j17Raced(t *testing.T, fx *consoleFixture, method, url, token, body string, hook func()) { |
| 93 | t.Helper() |
| 94 | var claim, rec string |
| 95 | j17Arm(fx, func() { |
| 96 | hook() |
| 97 | claim, rec = claimBytes(t, fx, "acme.go.tito.io"), j17RecordBytes(t, fx) |
| 98 | }) |
| 99 | serial := j17Serial(t, fx) |
| 100 | code, _, out := rawReq(t, method, url, token, body) |
| 101 | if claim == "" { |
| 102 | t.Fatal("the race hook never ran") |
| 103 | } |
| 104 | if code != http.StatusConflict || errorText(out) != moveChangedSentence { |
| 105 | t.Fatalf("raced door = %d %s", code, out) |
| 106 | } |
| 107 | if claimBytes(t, fx, "acme.go.tito.io") != claim { |
| 108 | t.Fatal("the raced door wrote the claim") |
| 109 | } |
| 110 | if j17RecordBytes(t, fx) != rec { |
| 111 | t.Fatal("the raced door mutated the successor record") |
| 112 | } |
| 113 | if j17Serial(t, fx) != serial { |
| 114 | t.Fatal("the raced door bumped the serial") |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | func j17RetryReady(t *testing.T, fx *consoleFixture) { |
| 119 | t.Helper() |
| 120 | ctx := context.Background() |
| 121 | obs, err := hostclaim.Read(ctx, fx.rel, "acme.go.tito.io") |
| 122 | if err != nil { |
| 123 | t.Fatal(err) |
| 124 | } |
| 125 | rb := "rb_abcdefghijklmnopqrstuvwxyz" |
| 126 | if _, err := hostclaim.Update(ctx, fx.rel, obs, func(c *hostclaim.Claim) error { |
| 127 | c.Op.Kind, c.Op.Phase = hostclaim.KindRestoring, hostclaim.PhaseProbing |
| 128 | c.Op.Intent["rollback"] = rb |
| 129 | return nil |
| 130 | }); err != nil { |
| 131 | t.Fatal(err) |
| 132 | } |
| 133 | j17SetRecord(t, fx, func(r *provision.Record) { |
| 134 | r.Move.State, r.Move.StuckReason = provision.MoveStuck, provision.StuckRecoveryProbeAttempts |
| 135 | r.Move.Rollback = &provision.MoveRollback{ID: rb, TargetPhase: provision.RbTargetReleased, SourcePhase: provision.RbSourceProbing, StuckAt: provision.RbSourceProbing} |
| 136 | }) |
| 137 | } |
| 138 | |
| 139 | func TestJ17DoorCutRefusesReplacedMove(t *testing.T) { |
| 140 | fx, token, moveURL, id := j17StartedMove(t, "code-j17-r2-cut") |
| 141 | j17SetRecord(t, fx, j17Ready) |
| 142 | j17Raced(t, fx, http.MethodPost, moveURL+"/cut", token, "", func() { j17Replace(t, fx, id) }) |
| 143 | } |
| 144 | |
| 145 | func TestJ17DoorAbortRefusesReplacedMove(t *testing.T) { |
| 146 | fx, token, moveURL, id := j17StartedMove(t, "code-j17-r2-abort") |
| 147 | j17Raced(t, fx, http.MethodDelete, moveURL, token, "", func() { j17Replace(t, fx, id) }) |
| 148 | } |
| 149 | |
| 150 | func TestJ17DoorRetryRefusesReplacedMove(t *testing.T) { |
| 151 | fx, token, moveURL, id := j17StartedMove(t, "code-j17-r2-retry") |
| 152 | j17RetryReady(t, fx) |
| 153 | j17Raced(t, fx, http.MethodPost, moveURL+"/resolve", token, `{"retry":true}`, func() { j17Replace(t, fx, id) }) |
| 154 | } |
| 155 | |
| 156 | // The same move and incarnation, the record advanced: the fresh observation |
| 157 | // carries a newer record_seq than the one the gate read. |
| 158 | func TestJ17DoorRefusesAdvancedRecordSeq(t *testing.T) { |
| 159 | advance := func(t *testing.T, fx *consoleFixture) func() { |
| 160 | return func() { |
| 161 | before := j17RecordBytes(t, fx) |
| 162 | j17SetRecord(t, fx, func(r *provision.Record) {}) |
| 163 | if j17RecordBytes(t, fx) == before { |
| 164 | t.Fatal("the record did not advance") |
| 165 | } |
| 166 | } |
| 167 | } |
| 168 | t.Run("cut", func(t *testing.T) { |
| 169 | fx, token, moveURL, _ := j17StartedMove(t, "code-j17-r2-seq-cut") |
| 170 | j17SetRecord(t, fx, j17Ready) |
| 171 | j17Raced(t, fx, http.MethodPost, moveURL+"/cut", token, "", advance(t, fx)) |
| 172 | }) |
| 173 | t.Run("abort", func(t *testing.T) { |
| 174 | fx, token, moveURL, _ := j17StartedMove(t, "code-j17-r2-seq-abort") |
| 175 | j17Raced(t, fx, http.MethodDelete, moveURL, token, "", advance(t, fx)) |
| 176 | }) |
| 177 | t.Run("retry", func(t *testing.T) { |
| 178 | fx, token, moveURL, _ := j17StartedMove(t, "code-j17-r2-seq-retry") |
| 179 | j17RetryReady(t, fx) |
| 180 | j17Raced(t, fx, http.MethodPost, moveURL+"/resolve", token, `{"retry":true}`, advance(t, fx)) |
| 181 | }) |
| 182 | } |
| 183 | |
| 184 | func TestJ17DoorRefusesReplacedIncarnation(t *testing.T) { |
| 185 | for _, action := range []string{"cut", "abort", "retry"} { |
| 186 | t.Run(action, func(t *testing.T) { |
| 187 | fx, token, url, id := j17StartedMove(t, "code-j17-r2-inc-"+action) |
| 188 | method, body := http.MethodDelete, "" |
| 189 | switch action { |
| 190 | case "cut": |
| 191 | j17SetRecord(t, fx, j17Ready) |
| 192 | method, url = http.MethodPost, url+"/cut" |
| 193 | case "retry": |
| 194 | j17RetryReady(t, fx) |
| 195 | method, url, body = http.MethodPost, url+"/resolve", `{"retry":true}` |
| 196 | } |
| 197 | j17Raced(t, fx, method, url, token, body, func() { |
| 198 | ctx := context.Background() |
| 199 | r, tag, err := provision.GetRecordETag(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io") |
| 200 | if err != nil { |
| 201 | t.Fatal(err) |
| 202 | } |
| 203 | r.CreatedAt = r.CreatedAt.Add(time.Second) |
| 204 | inc, err := hostclaim.Incarnation(r.Account, r.App, r.Host, r.CreatedAt) |
| 205 | if err != nil { |
| 206 | t.Fatal(err) |
| 207 | } |
| 208 | r.Move.Incarnation = inc |
| 209 | b, _ := json.Marshal(r) |
| 210 | if _, err := fx.rel.Put(ctx, provision.RecordKey(r.Account, r.App, r.Host), b, tag); err != nil { |
| 211 | t.Fatal(err) |
| 212 | } |
| 213 | o, err := hostclaim.Read(ctx, fx.rel, r.Host) |
| 214 | if err != nil { |
| 215 | t.Fatal(err) |
| 216 | } |
| 217 | if _, err := hostclaim.Update(ctx, fx.rel, o, func(c *hostclaim.Claim) error { |
| 218 | c.Holder.Incarnation = inc |
| 219 | c.Holder.CreatedAt = r.CreatedAt |
| 220 | return nil |
| 221 | }); err != nil { |
| 222 | t.Fatal(err) |
| 223 | } |
| 224 | a, err := hostops.ReadAdmission(ctx, fx.rel, "bab", "titogo", claimMoveAppID) |
| 225 | if err != nil { |
| 226 | t.Fatal(err) |
| 227 | } |
| 228 | reg := a.Value.Registrations[id] |
| 229 | reg.Incarnation = inc |
| 230 | a.Value.Registrations[id] = reg |
| 231 | b, _ = json.Marshal(a.Value) |
| 232 | if _, err := fx.rel.Put(ctx, a.Key, b, a.ETag); err != nil { |
| 233 | t.Fatal(err) |
| 234 | } |
| 235 | }) |
| 236 | }) |
| 237 | } |
| 238 | } |
| 239 | |
amadan