| 1 | package release |
| 2 | |
| 3 | import ( |
| 4 | "os" |
| 5 | "path/filepath" |
| 6 | "reflect" |
| 7 | "testing" |
| 8 | "time" |
| 9 | |
| 10 | "github.com/carlosframework/platform/internal/registry" |
| 11 | ) |
| 12 | |
| 13 | func TestBuildStateKeySitsBesideTheLock(t *testing.T) { |
| 14 | if got := BuildStateKey("instances/a.example.com"); got != "instances/a.example.com/build-state.json" { |
| 15 | t.Fatalf("BuildStateKey = %q", got) |
| 16 | } |
| 17 | } |
| 18 | |
| 19 | func TestConsumedRoundTrip(t *testing.T) { |
| 20 | opt := t.TempDir() |
| 21 | if c, err := ReadConsumed(opt, "a.example.com"); c != nil || err != nil { |
| 22 | t.Fatalf("absent = %+v, %v", c, err) |
| 23 | } |
| 24 | want := Consumed{Schema: 1, Host: "a.example.com", Resolved: &ResolvedRequest{ID: 3, Target: "v2"}, Completed: 2, |
| 25 | Desired: "v2", Reason: string(ReasonRequest), At: time.Now().UTC().Truncate(time.Second)} |
| 26 | if err := WriteConsumed(opt, want); err != nil { |
| 27 | t.Fatal(err) |
| 28 | } |
| 29 | got, err := ReadConsumed(opt, "a.example.com") |
| 30 | if err != nil || got.Resolved.Target != "v2" || got.Completed != 2 || got.Reason != string(ReasonRequest) { |
| 31 | t.Fatalf("round trip = %+v, %v", got, err) |
| 32 | } |
| 33 | if ConsumedPath(opt, "a.example.com") != filepath.Join(opt, "consumed", "a.example.com") { |
| 34 | t.Fatal("the consumed file moved") |
| 35 | } |
| 36 | } |
| 37 | |
| 38 | func TestConsumedRefusesAnotherHostsFile(t *testing.T) { |
| 39 | opt := t.TempDir() |
| 40 | if err := WriteConsumed(opt, Consumed{Schema: 1, Host: "b.example.com"}); err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | if err := WriteFileAtomic(ConsumedPath(opt, "a.example.com"), mustRead(t, ConsumedPath(opt, "b.example.com")), 0o644); err != nil { |
| 44 | t.Fatal(err) |
| 45 | } |
| 46 | if _, err := ReadConsumed(opt, "a.example.com"); err == nil { |
| 47 | t.Fatal("a consumed file naming another host was accepted") |
| 48 | } |
| 49 | } |
| 50 | |
| 51 | func TestWatermarkIsTheHighestOfBoth(t *testing.T) { |
| 52 | for _, c := range []struct { |
| 53 | c *Consumed |
| 54 | b *BuildState |
| 55 | want uint64 |
| 56 | }{ |
| 57 | {nil, nil, 0}, |
| 58 | {&Consumed{Completed: 4}, nil, 4}, |
| 59 | {nil, &BuildState{Completed: 6}, 6}, |
| 60 | {&Consumed{Completed: 4}, &BuildState{Completed: 6}, 6}, |
| 61 | {&Consumed{Completed: 9}, &BuildState{Completed: 6}, 9}, |
| 62 | } { |
| 63 | if got := Watermark(c.c, c.b); got != c.want { |
| 64 | t.Errorf("Watermark(%+v, %+v) = %d, want %d", c.c, c.b, got, c.want) |
| 65 | } |
| 66 | } |
| 67 | } |
| 68 | |
| 69 | func TestBuildStateCopyColumn(t *testing.T) { |
| 70 | c, err := ParseBuildStateCopy("") |
| 71 | if err != nil || c.Known { |
| 72 | t.Fatalf("an empty column is unknown: %+v %v", c, err) |
| 73 | } |
| 74 | in := BuildStateCopy{Known: true, Replica: true, State: &BuildState{Schema: 1, Host: "a.example.com", LastExecuted: "v1"}} |
| 75 | out, err := ParseBuildStateCopy(in.Column()) |
| 76 | if err != nil || !out.Known || !out.Replica || out.State.LastExecuted != "v1" { |
| 77 | t.Fatalf("round trip = %+v %v", out, err) |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | func TestNextBuildState(t *testing.T) { |
| 82 | for _, tc := range []struct { |
| 83 | name string |
| 84 | cur *BuildState |
| 85 | c *Consumed |
| 86 | build string |
| 87 | want BuildState |
| 88 | }{ |
| 89 | {"a: nil cur, nil consumed sets only LastExecuted", nil, nil, "v1", |
| 90 | BuildState{LastExecuted: "v1"}}, |
| 91 | {"b: a consumed resolution above the watermark is carried and completes when its target runs", |
| 92 | &BuildState{Schema: 1, Host: "h", Completed: 2}, |
| 93 | &Consumed{Resolved: &ResolvedRequest{ID: 3, Target: "v3"}, Completed: 2}, "v3", |
| 94 | BuildState{Schema: 1, Host: "h", LastExecuted: "v3", Resolved: &ResolvedRequest{ID: 3, Target: "v3"}, Completed: 3}}, |
| 95 | {"c: running another build carries the resolution but does not complete it", |
| 96 | &BuildState{Schema: 1, Host: "h", Completed: 2}, |
| 97 | &Consumed{Resolved: &ResolvedRequest{ID: 3, Target: "v3"}, Completed: 2}, "v2", |
| 98 | BuildState{Schema: 1, Host: "h", LastExecuted: "v2", Resolved: &ResolvedRequest{ID: 3, Target: "v3"}, Completed: 2}}, |
| 99 | {"d: a consumed resolution at or below the state's Completed is not carried", |
| 100 | &BuildState{Schema: 1, Host: "h", Completed: 3}, |
| 101 | &Consumed{Resolved: &ResolvedRequest{ID: 3, Target: "v3"}, Completed: 2}, "v3", |
| 102 | BuildState{Schema: 1, Host: "h", LastExecuted: "v3", Completed: 3}}, |
| 103 | {"e: a higher consumed Completed raises the state's and carries CompletedFailed", |
| 104 | &BuildState{Schema: 1, Host: "h", Completed: 2}, |
| 105 | &Consumed{Completed: 5, CompletedFailed: true}, "v1", |
| 106 | BuildState{Schema: 1, Host: "h", LastExecuted: "v1", Completed: 5, CompletedFailed: true}}, |
| 107 | {"f: an equal Completed with consumed CompletedFailed marks the state failed and does not reopen", |
| 108 | &BuildState{Schema: 1, Host: "h", Completed: 4}, |
| 109 | &Consumed{Completed: 4, CompletedFailed: true}, "v1", |
| 110 | BuildState{Schema: 1, Host: "h", LastExecuted: "v1", Completed: 4, CompletedFailed: true}}, |
| 111 | {"g: the state's own resolution from another box completes when its target runs", |
| 112 | &BuildState{Schema: 1, Host: "h", Resolved: &ResolvedRequest{ID: 4, Target: "v3"}, Completed: 3}, |
| 113 | nil, "v3", |
| 114 | BuildState{Schema: 1, Host: "h", LastExecuted: "v3", Resolved: &ResolvedRequest{ID: 4, Target: "v3"}, Completed: 4}}, |
| 115 | {"h: the watermark never decreases", |
| 116 | &BuildState{Schema: 1, Host: "h", Completed: 6}, |
| 117 | &Consumed{Completed: 4}, "v1", |
| 118 | BuildState{Schema: 1, Host: "h", LastExecuted: "v1", Completed: 6}}, |
| 119 | {"i: a resolution below the consumed Completed never lowers the watermark", |
| 120 | &BuildState{Schema: 1, Host: "h", Resolved: &ResolvedRequest{ID: 3, Target: "v3"}, Completed: 2}, |
| 121 | &Consumed{Completed: 5, CompletedFailed: true}, "v3", |
| 122 | BuildState{Schema: 1, Host: "h", LastExecuted: "v3", Resolved: &ResolvedRequest{ID: 3, Target: "v3"}, Completed: 5, CompletedFailed: true}}, |
| 123 | {"ii: a closed request whose target runs again stays closed and failed", |
| 124 | &BuildState{Schema: 1, Host: "h", Resolved: &ResolvedRequest{ID: 3, Target: "v3"}, Completed: 3, CompletedFailed: true}, |
| 125 | nil, "v3", |
| 126 | BuildState{Schema: 1, Host: "h", LastExecuted: "v3", Resolved: &ResolvedRequest{ID: 3, Target: "v3"}, Completed: 3, CompletedFailed: true}}, |
| 127 | {"j: an older resolution in this box's consumed file does not replace the state's newer one", |
| 128 | &BuildState{Schema: 1, Host: "h", Resolved: &ResolvedRequest{ID: 5, Target: "v5"}, Completed: 3}, |
| 129 | &Consumed{Resolved: &ResolvedRequest{ID: 4, Target: "v4"}, Completed: 3}, "v5", |
| 130 | BuildState{Schema: 1, Host: "h", LastExecuted: "v5", Resolved: &ResolvedRequest{ID: 5, Target: "v5"}, Completed: 5}}, |
| 131 | } { |
| 132 | t.Run(tc.name, func(t *testing.T) { |
| 133 | var curBefore BuildState |
| 134 | if tc.cur != nil { |
| 135 | curBefore = *tc.cur |
| 136 | } |
| 137 | got := NextBuildState(tc.cur, tc.c, tc.build) |
| 138 | if !reflect.DeepEqual(got, tc.want) { |
| 139 | t.Fatalf("NextBuildState = %+v (resolved %+v), want %+v (resolved %+v)", got, got.Resolved, tc.want, tc.want.Resolved) |
| 140 | } |
| 141 | if tc.cur != nil && !reflect.DeepEqual(*tc.cur, curBefore) { |
| 142 | t.Fatalf("cur was mutated: %+v", *tc.cur) |
| 143 | } |
| 144 | }) |
| 145 | } |
| 146 | } |
| 147 | |
| 148 | func TestNextBuildStateDoesNotAliasConsumed(t *testing.T) { |
| 149 | c := &Consumed{Resolved: &ResolvedRequest{ID: 3, Target: "v3"}} |
| 150 | got := NextBuildState(nil, c, "v2") |
| 151 | c.Resolved.Target = "changed" |
| 152 | c.Resolved.ID = 99 |
| 153 | if got.Resolved == nil || got.Resolved.Target != "v3" || got.Resolved.ID != 3 { |
| 154 | t.Fatalf("the state aliases the consumed file: %+v", got.Resolved) |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | func mustRead(t *testing.T, path string) []byte { |
| 159 | t.Helper() |
| 160 | b, err := os.ReadFile(path) |
| 161 | if err != nil { |
| 162 | t.Fatal(err) |
| 163 | } |
| 164 | return b |
| 165 | } |
| 166 | |
| 167 | // Ruling R-SPEC-43A-2 (Astra round 1 finding 3): a graceful move carries the |
| 168 | // source box's consumed request record into the build state: root's |
| 169 | // resolution when it is newer than everything the state knows, and a failed |
| 170 | // completion; never a successful completion, and never last_executed (only a |
| 171 | // run changes that). |
| 172 | func TestCarryRequestCarriesTheResolutionAndAFailedCompletion(t *testing.T) { |
| 173 | cur := &BuildState{Schema: 1, Host: "h", LastExecuted: "A", Completed: 1} |
| 174 | c := &Consumed{Host: "h", Resolved: &ResolvedRequest{ID: 3, Target: "T"}, Completed: 2, CompletedFailed: true} |
| 175 | got := CarryRequest(cur, c) |
| 176 | if got.LastExecuted != "A" || got.Completed != 2 || !got.CompletedFailed || got.Resolved == nil || got.Resolved.ID != 3 || got.Resolved.Target != "T" { |
| 177 | t.Fatalf("carried %+v; want A, completed 2 failed, {3 T}", got) |
| 178 | } |
| 179 | if cur.Completed != 1 || cur.Resolved != nil { |
| 180 | t.Fatalf("the input was changed: %+v", cur) |
| 181 | } |
| 182 | // A successful completion is the pre-exec write's alone. |
| 183 | if got := CarryRequest(cur, &Consumed{Host: "h", Completed: 5}); got.Completed != 1 || got.CompletedFailed { |
| 184 | t.Fatalf("a successful completion was carried: %+v", got) |
| 185 | } |
| 186 | // An older resolution never replaces a newer one; a same-id failure marks |
| 187 | // the completed request failed without reopening it. |
| 188 | newer := &BuildState{LastExecuted: "A", Completed: 2, Resolved: &ResolvedRequest{ID: 4, Target: "U"}} |
| 189 | if got := CarryRequest(newer, &Consumed{Resolved: &ResolvedRequest{ID: 3, Target: "T"}, Completed: 2, CompletedFailed: true}); got.Resolved.ID != 4 || got.Completed != 2 || !got.CompletedFailed { |
| 190 | t.Fatalf("carried %+v; want {4 U} kept, R2 completed failed", got) |
| 191 | } |
| 192 | // No state yet: the record alone makes one, with no last_executed. |
| 193 | if got := CarryRequest(nil, c); got.LastExecuted != "" || got.Completed != 2 || !got.CompletedFailed || got.Resolved == nil { |
| 194 | t.Fatalf("carried onto no state: %+v", got) |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | // The scenario: a sleeping manual instance last ran A. Root resolves R1 to |
| 199 | // T, which this box holds as failed, so R1 completes failed in root's file |
| 200 | // and nothing runs. A graceful move carries that into the build state, and |
| 201 | // the destination never resolves R1 again (rule 2: a completed request never |
| 202 | // re-enters), so it installs nothing new without another Update now. A box |
| 203 | // that is lost carries nothing: the destination then resolves R1 again, the |
| 204 | // spec's lost request (§4.2). |
| 205 | func TestAFailedCompletionSurvivesAGracefulMove(t *testing.T) { |
| 206 | u := UpdateSettings{Updates: UpdatesManual, Channel: "stable", InstallRequest: 1} |
| 207 | route := registry.Route{Host: "h", Kind: "instance", UpdateSettings: u.Column()} |
| 208 | state := &BuildState{Schema: 1, Host: "h", LastExecuted: "A"} |
| 209 | src := TargetInput{Route: route, Pointer: &ChannelPointer{Version: "T"}, Settings: u, |
| 210 | Build: BuildStateCopy{Known: true, Replica: true, State: state}, Installed: "A", |
| 211 | Failed: func(v string) bool { return v == "T" }} |
| 212 | dec := Target(src) |
| 213 | if dec.Resolve == nil || dec.Fail != 1 { |
| 214 | t.Fatalf("source decision %+v; want R1 resolved to T and completed failed", dec) |
| 215 | } |
| 216 | consumed := NextConsumed(src, dec) |
| 217 | dst := func(st *BuildState) TargetDecision { |
| 218 | return Target(TargetInput{Route: route, Pointer: &ChannelPointer{Version: "H"}, Settings: u, |
| 219 | Build: BuildStateCopy{Known: true, Replica: true, State: st}}) |
| 220 | } |
| 221 | carried := CarryRequest(state, &consumed) |
| 222 | if d := dst(&carried); d.Resolve != nil || d.Reason == ReasonRequest || d.Version != "A" { |
| 223 | t.Fatalf("destination after a graceful move: %+v; want R1 closed, held on A", d) |
| 224 | } |
| 225 | if d := dst(state); d.Resolve == nil || d.Resolve.ID != 1 || d.Version != "H" { |
| 226 | t.Fatalf("destination after a lost box: %+v; want R1 resolved again (a lost request)", d) |
| 227 | } |
| 228 | } |
| 229 | |
| 230 | // Found by the simulation (seed 164 after Astra round 1's changes): a box |
| 231 | // the instance returns to keeps its old row, and that copy's refreshes |
| 232 | // failed while the instance was away (ReadErr), so it still says request R1 |
| 233 | // completed while the bucket says R5 completed failed. Root must read such a |
| 234 | // copy as unread (R-TARGET-UNKNOWN): no request bookkeeping, so R5 is not |
| 235 | // resolved again from a stale watermark, and a held instance is unknown |
| 236 | // rather than recovered onto a stale last_executed. |
| 237 | func TestTargetReadsACopyWhoseRefreshFailedAsUnread(t *testing.T) { |
| 238 | stale := BuildStateCopy{Known: true, Replica: true, ReadErr: "s3: timeout", State: &BuildState{LastExecuted: "b1", Completed: 1, CompletedFailed: true}} |
| 239 | for _, u := range []UpdateSettings{ |
| 240 | {Updates: UpdatesAuto, Channel: "stable", InstallRequest: 5}, |
| 241 | {Updates: UpdatesManual, Channel: "stable", InstallRequest: 5}, |
| 242 | } { |
| 243 | in := TargetInput{Route: registry.Route{Host: "h", UpdateSettings: u.Column()}, Pointer: &ChannelPointer{Version: "b6"}, Settings: u, |
| 244 | Build: stale, Consumed: &Consumed{Host: "h", Completed: 1, CompletedFailed: true}, Installed: "b1"} |
| 245 | d := Target(in) |
| 246 | if d.Resolve != nil || d.Reason == ReasonRequest { |
| 247 | t.Fatalf("%s: %+v; a stale copy's watermark resolved R5 again", u.Updates, d) |
| 248 | } |
| 249 | if u.Updates == UpdatesManual && (d.Reason != ReasonUnknown || d.Version != "") { |
| 250 | t.Fatalf("manual: %+v; want unknown, nothing installed", d) |
| 251 | } |
| 252 | if next := NextConsumed(in, d); next.Baking != "" || (next.Resolved != nil && next.Resolved.ID == 5) { |
| 253 | t.Fatalf("%s: consumed %+v", u.Updates, next) |
| 254 | } |
| 255 | } |
| 256 | } |
| 257 | |
amadan