| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "errors" |
| 5 | "testing" |
| 6 | |
| 7 | "github.com/carlosframework/platform/internal/registry" |
| 8 | "github.com/carlosframework/platform/internal/release" |
| 9 | "github.com/carlosframework/platform/internal/status" |
| 10 | ) |
| 11 | |
| 12 | func TestTheReportCarriesRootsDecision(t *testing.T) { |
| 13 | t.Parallel() |
| 14 | b, _, reg := testBuildStates(t) |
| 15 | if err := release.WriteConsumed(b.optDir, release.Consumed{Schema: 1, Host: "a.example.com", Account: release.OpsAccount, App: "km", Desired: "v1", Reason: "major", Detail: "v1", Completed: 2, Baking: "v4"}); err != nil { |
| 16 | t.Fatal(err) |
| 17 | } |
| 18 | two := 2 |
| 19 | if err := reg.SetUpdateSettings("a.example.com", release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "stable", Major: &two, InstallRequest: 3}.Column()); err != nil { |
| 20 | t.Fatal(err) |
| 21 | } |
| 22 | got := updatesReport(reg, b.optDir)["a.example.com"] |
| 23 | if got.Desired != "v1" || got.Reason != "major" || got.Detail != "v1" || got.Pending != 3 || got.Completed != 2 || |
| 24 | got.Baking != "v4" || got.Major == nil || *got.Major != 2 { |
| 25 | t.Fatalf("report = %+v; want root's decision, the baking head beside it, and the effective major", got) |
| 26 | } |
| 27 | rep, err := updatesReader(reg, b.optDir)("a.example.com") |
| 28 | if err != nil || rep.Desired != "v1" || rep.Reason != "major" || rep.Settings.Major == nil { |
| 29 | t.Fatalf("socket report = %+v, %v", rep, err) |
| 30 | } |
| 31 | } |
| 32 | |
| 33 | // Astra r1 (R18): each entry names the instance's tenant from the box's own |
| 34 | // route, as the rollout report does (release.RouteAccount), so the console |
| 35 | // never needs the rollout report to tell whose report it is. |
| 36 | func TestTheReportNamesTheTenant(t *testing.T) { |
| 37 | t.Parallel() |
| 38 | b, _, reg := testBuildStates(t) |
| 39 | if err := reg.Upsert(registry.Route{Host: "b.example.com", Kind: "instance", Account: "bab", App: "shop", Hibernate: true, Socket: "/run/carlos/b.sock"}); err != nil { |
| 40 | t.Fatal(err) |
| 41 | } |
| 42 | // Root names the tenant it decided for (R22); the report checks it. |
| 43 | acct := map[string][2]string{"a.example.com": {release.OpsAccount, "km"}, "b.example.com": {"bab", "shop"}} |
| 44 | for _, h := range []string{"a.example.com", "b.example.com"} { |
| 45 | if err := release.WriteConsumed(b.optDir, release.Consumed{Schema: 1, Host: h, Account: acct[h][0], App: acct[h][1], Desired: "v1", Reason: "manual"}); err != nil { |
| 46 | t.Fatal(err) |
| 47 | } |
| 48 | } |
| 49 | got := updatesReport(reg, b.optDir) |
| 50 | if u := got["b.example.com"]; u.Account != "bab" || u.App != "shop" || u.Reason != "manual" { |
| 51 | t.Fatalf("b = %+v; want tenant bab/shop", u) |
| 52 | } |
| 53 | // A route with no account of its own is the platform's, as everywhere. |
| 54 | if u := got["a.example.com"]; u.Account != release.OpsAccount || u.App != "km" { |
| 55 | t.Fatalf("a = %+v; want tenant %s/km", u, release.OpsAccount) |
| 56 | } |
| 57 | } |
| 58 | |
| 59 | func TestEveryBoxOfThisBinaryReportsTheProtocol(t *testing.T) { |
| 60 | if m := boxSettingsMetrics(); m != status.UpdateSettingsProtocol { |
| 61 | t.Fatalf("this binary reports %d", m) |
| 62 | } |
| 63 | // Astra I3 #1: a box whose build-state service failed does not claim to |
| 64 | // enforce settings, so the console keeps refusing them. |
| 65 | buildStatesFailed.Store(true) |
| 66 | t.Cleanup(func() { buildStatesFailed.Store(false) }) |
| 67 | if m := boxSettingsMetrics(); m != 0 { |
| 68 | t.Fatalf("a box with no build-state service reports %d", m) |
| 69 | } |
| 70 | } |
| 71 | |
| 72 | // The agent's note of a failed build-state service is what the report reads: |
| 73 | // a box with no replica bucket (nil, nil) has nothing to enforce and reports |
| 74 | // the protocol; one whose store failed to start reports 0. |
| 75 | func TestTheProtocolFollowsTheBuildStateService(t *testing.T) { |
| 76 | t.Cleanup(func() { buildStatesFailed.Store(false) }) |
| 77 | noteBuildStatesInit(nil) |
| 78 | if m := boxSettingsMetrics(); m != status.UpdateSettingsProtocol { |
| 79 | t.Fatalf("a box with no replica bucket reports %d, want the protocol", m) |
| 80 | } |
| 81 | noteBuildStatesInit(errors.New("no regions")) |
| 82 | if m := boxSettingsMetrics(); m != 0 { |
| 83 | t.Fatalf("a box whose build-state store failed reports %d, want 0", m) |
| 84 | } |
| 85 | noteBuildStatesInit(nil) |
| 86 | if m := boxSettingsMetrics(); m != status.UpdateSettingsProtocol { |
| 87 | t.Fatalf("a recovered start reports %d, want the protocol", m) |
| 88 | } |
| 89 | } |
| 90 | |
| 91 | func TestTheStatusProbesCarryTheProtocolAndTheDecisions(t *testing.T) { |
| 92 | t.Parallel() |
| 93 | b, _, reg := testBuildStates(t) |
| 94 | if err := release.WriteConsumed(b.optDir, release.Consumed{Schema: 1, Host: "a.example.com", Account: release.OpsAccount, App: "km", Desired: "v1", Reason: "manual"}); err != nil { |
| 95 | t.Fatal(err) |
| 96 | } |
| 97 | var p status.Probes |
| 98 | fillUpdateProbes(&p, reg, b.optDir) |
| 99 | if p.UpdateSettings != status.UpdateSettingsProtocol || p.Updates["a.example.com"].Reason != "manual" { |
| 100 | t.Fatalf("probes = %d, %+v", p.UpdateSettings, p.Updates) |
| 101 | } |
| 102 | } |
| 103 | |
| 104 | // A newer console may add a Control name this binary does not know; the |
| 105 | // report must still be made, with the hold it carries (R-SETTINGS-UNREADABLE). |
| 106 | func TestTheReportSurvivesAnUnknownControlName(t *testing.T) { |
| 107 | t.Parallel() |
| 108 | b, _, reg := testBuildStates(t) |
| 109 | if err := release.WriteConsumed(b.optDir, release.Consumed{Schema: 1, Host: "a.example.com", Account: release.OpsAccount, App: "km", Desired: "v1", Reason: "manual"}); err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | col := `{"updates":"manual","channel":"stable","control":["updates","from-the-future"]}` |
| 113 | if err := reg.SetUpdateSettings("a.example.com", col); err != nil { |
| 114 | t.Fatal(err) |
| 115 | } |
| 116 | if got := updatesReport(reg, b.optDir)["a.example.com"]; got.Reason != "manual" { |
| 117 | t.Fatalf("report = %+v; want the decision despite the unknown control name", got) |
| 118 | } |
| 119 | rep, err := updatesReader(reg, b.optDir)("a.example.com") |
| 120 | if err != nil || rep.Reason != "manual" || rep.Settings.Updates != release.UpdatesManual || rep.Settings.Channel != "stable" { |
| 121 | t.Fatalf("socket report = %+v, %v", rep, err) |
| 122 | } |
| 123 | if len(rep.MayChange) != 1 || rep.MayChange[0] != release.ControlUpdates { |
| 124 | t.Fatalf("may change = %v; want only the known name", rep.MayChange) |
| 125 | } |
| 126 | } |
| 127 | |
| 128 | // A route with the default settings and no decision has nothing to say; the |
| 129 | // serving build is added to the socket report; the channel fallback stays. |
| 130 | func TestTheReaderKeepsTheChannelFallbackAndAddsTheServingBuild(t *testing.T) { |
| 131 | t.Parallel() |
| 132 | b, _, reg := testBuildStates(t) |
| 133 | if got := updatesReport(reg, b.optDir); got != nil { |
| 134 | t.Fatalf("report = %+v; want none for a default instance root has not decided", got) |
| 135 | } |
| 136 | rt, _ := reg.Get("a.example.com") |
| 137 | rt.Channel = "beta" |
| 138 | if err := reg.Upsert(*rt); err != nil { |
| 139 | t.Fatal(err) |
| 140 | } |
| 141 | if err := reg.SetServingVersion("a.example.com", "v7"); err != nil { |
| 142 | t.Fatal(err) |
| 143 | } |
| 144 | rep, err := updatesReader(reg, b.optDir)("a.example.com") |
| 145 | if err != nil || rep.Settings.Channel != "beta" || rep.Serving != "v7" { |
| 146 | t.Fatalf("socket report = %+v, %v; want the route's channel and its serving build", rep, err) |
| 147 | } |
| 148 | } |
| 149 | |
| 150 | // Consumed.Detail for reason unknown can be a raw parse error; it must not |
| 151 | // reach a member. A build version is the one detail that may. |
| 152 | func TestAnUnknownReasonReportsNoRawDetail(t *testing.T) { |
| 153 | t.Parallel() |
| 154 | b, _, reg := testBuildStates(t) |
| 155 | raw := `update settings: invalid character 'x' looking for beginning of value` |
| 156 | for _, c := range []struct{ detail, want string }{{raw, ""}, {"v3", "v3"}} { |
| 157 | if err := release.WriteConsumed(b.optDir, release.Consumed{Schema: 1, Host: "a.example.com", Account: release.OpsAccount, App: "km", Reason: "unknown", Detail: c.detail}); err != nil { |
| 158 | t.Fatal(err) |
| 159 | } |
| 160 | got := updatesReport(reg, b.optDir)["a.example.com"] |
| 161 | if got.Reason != "unknown" || got.Detail != c.want { |
| 162 | t.Fatalf("detail %q: report = %+v, want detail %q", c.detail, got, c.want) |
| 163 | } |
| 164 | } |
| 165 | if err := release.WriteConsumed(b.optDir, release.Consumed{Schema: 1, Host: "a.example.com", Account: release.OpsAccount, App: "km", Reason: "major", Detail: "not a version!"}); err != nil { |
| 166 | t.Fatal(err) |
| 167 | } |
| 168 | if got := updatesReport(reg, b.optDir)["a.example.com"]; got.Detail != "not a version!" { |
| 169 | t.Fatalf("a reason other than unknown keeps root's detail, got %+v", got) |
| 170 | } |
| 171 | } |
| 172 | |
| 173 | // The watermark joins the consumed file and the build state only when the |
| 174 | // build-state copy has been read. Unread, the report is the consumed file's |
| 175 | // alone, and a request is pending only above that. |
| 176 | func TestAnUnreadBuildStateReportsTheConsumedFileAlone(t *testing.T) { |
| 177 | t.Parallel() |
| 178 | b, _, reg := testBuildStates(t) |
| 179 | if err := release.WriteConsumed(b.optDir, release.Consumed{Schema: 1, Host: "a.example.com", Account: release.OpsAccount, App: "km", Reason: "request", Completed: 2}); err != nil { |
| 180 | t.Fatal(err) |
| 181 | } |
| 182 | set := func(req uint64) { |
| 183 | t.Helper() |
| 184 | if err := reg.SetUpdateSettings("a.example.com", release.UpdateSettings{Updates: release.UpdatesAuto, InstallRequest: req}.Column()); err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | } |
| 188 | setCopy := func(c release.BuildStateCopy) { |
| 189 | t.Helper() |
| 190 | if err := reg.SetBuildState("a.example.com", c.Column()); err != nil { |
| 191 | t.Fatal(err) |
| 192 | } |
| 193 | } |
| 194 | // Unread copy that nevertheless holds a higher completed id: not used. |
| 195 | setCopy(release.BuildStateCopy{Known: false, State: &release.BuildState{Schema: 1, Host: "a.example.com", Completed: 9, CompletedFailed: true}}) |
| 196 | set(5) |
| 197 | got := updatesReport(reg, b.optDir)["a.example.com"] |
| 198 | if got.Completed != 2 || got.Pending != 5 || got.CompletedFailed { |
| 199 | t.Fatalf("unread: report = %+v; want completed 2 from the consumed file, pending 5", got) |
| 200 | } |
| 201 | set(2) |
| 202 | if got := updatesReport(reg, b.optDir)["a.example.com"]; got.Pending != 0 || got.Completed != 2 { |
| 203 | t.Fatalf("unread, request at the watermark: report = %+v; want nothing pending", got) |
| 204 | } |
| 205 | // Read: the build state's higher completed id counts. |
| 206 | setCopy(release.BuildStateCopy{Known: true, State: &release.BuildState{Schema: 1, Host: "a.example.com", Completed: 9, CompletedFailed: true}}) |
| 207 | set(5) |
| 208 | got = updatesReport(reg, b.optDir)["a.example.com"] |
| 209 | if got.Completed != 9 || got.Pending != 0 || !got.CompletedFailed { |
| 210 | t.Fatalf("read: report = %+v; want completed 9, failed, nothing pending", got) |
| 211 | } |
| 212 | } |
| 213 | |
| 214 | // Astra r2 N1 (ruling R22): the consumed file is keyed by hostname only. A |
| 215 | // hostname that moves from tenant A to tenant B on one box keeps A's file |
| 216 | // until root decides for B; the report labels its entry with B's route, so |
| 217 | // it must carry none of A's decision. B's own column (its major) stays. |
| 218 | func TestAMovedHostnameReportsNoPreviousTenantsDecision(t *testing.T) { |
| 219 | t.Parallel() |
| 220 | b, _, reg := testBuildStates(t) |
| 221 | const host = "a.example.com" |
| 222 | three, five := 3, 5 |
| 223 | // Tenant A's route, and root's real write for it. |
| 224 | a := registry.Route{Host: host, Kind: "instance", Account: "cdf", App: "shop", Hibernate: true, Socket: "/run/carlos/a.sock"} |
| 225 | if err := reg.Upsert(a); err != nil { |
| 226 | t.Fatal(err) |
| 227 | } |
| 228 | ua := release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "stable", Major: &five, InstallRequest: 6} |
| 229 | if err := reg.SetUpdateSettings(host, ua.Column()); err != nil { |
| 230 | t.Fatal(err) |
| 231 | } |
| 232 | a.UpdateSettings = ua.Column() |
| 233 | in := release.TargetInput{Route: a, Settings: ua, Pointer: &release.ChannelPointer{Channel: "stable", Version: "v-a-head"}, HeadHeld: "baking", |
| 234 | Build: release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{Schema: 1, Host: host}}} |
| 235 | if err := release.WriteConsumed(b.optDir, release.NextConsumed(in, release.TargetDecision{Version: "v-a-pin", Reason: release.ReasonPin, Detail: "v-a-pin", Fail: 4})); err != nil { |
| 236 | t.Fatal(err) |
| 237 | } |
| 238 | if got := updatesReport(reg, b.optDir)[host]; got.Desired != "v-a-pin" || got.Baking != "v-a-head" || got.Completed != 4 { |
| 239 | t.Fatalf("A's own report = %+v; want A's decision (the fixture is wrong otherwise)", got) |
| 240 | } |
| 241 | // The route is retired (claims off: the row is deleted) and B's arrives on |
| 242 | // the same hostname before root has decided for it. |
| 243 | if err := reg.Delete(host); err != nil { |
| 244 | t.Fatal(err) |
| 245 | } |
| 246 | if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Account: "dee", App: "blog", Hibernate: true, Socket: "/run/carlos/a.sock"}); err != nil { |
| 247 | t.Fatal(err) |
| 248 | } |
| 249 | ub := release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable", Major: &three, InstallRequest: 2} |
| 250 | if err := reg.SetUpdateSettings(host, ub.Column()); err != nil { |
| 251 | t.Fatal(err) |
| 252 | } |
| 253 | if err := reg.SetBuildState(host, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{Schema: 1, Host: host, Completed: 1}}.Column()); err != nil { |
| 254 | t.Fatal(err) |
| 255 | } |
| 256 | got, ok := updatesReport(reg, b.optDir)[host] |
| 257 | if !ok { |
| 258 | t.Fatal("B's route has settings of its own and no entry") |
| 259 | } |
| 260 | if got.Account != "dee" || got.App != "blog" { |
| 261 | t.Fatalf("B's entry names %s/%s, want dee/blog", got.Account, got.App) |
| 262 | } |
| 263 | if got.Desired != "" || got.Reason != "" || got.Detail != "" || got.Baking != "" || |
| 264 | got.Completed != 0 || got.CompletedFailed || got.Pending != 0 { |
| 265 | t.Fatalf("B's entry carries a decision root did not make for B: %+v", got) |
| 266 | } |
| 267 | if got.Major == nil || *got.Major != 3 { |
| 268 | t.Fatalf("B's entry major = %v, want B's own 3", got.Major) |
| 269 | } |
| 270 | rep, err := updatesReader(reg, b.optDir)(host) |
| 271 | if err != nil || rep.Desired != "" || rep.Reason != "" { |
| 272 | t.Fatalf("B's socket report = %+v, %v; want none of A's decision", rep, err) |
| 273 | } |
| 274 | if rep.Settings.Updates != release.UpdatesManual || rep.Settings.Major == nil || *rep.Settings.Major != 3 { |
| 275 | t.Fatalf("B's socket report settings = %+v, want B's own", rep.Settings) |
| 276 | } |
| 277 | |
| 278 | // Root's next pass decides for B, over A's file: now the decision is B's. |
| 279 | b2 := registry.Route{Host: host, Kind: "instance", Account: "dee", App: "blog", UpdateSettings: ub.Column()} |
| 280 | prev, err := release.ReadConsumed(b.optDir, host) |
| 281 | if err != nil { |
| 282 | t.Fatal(err) |
| 283 | } |
| 284 | inB := release.TargetInput{Route: b2, Settings: ub, Consumed: prev, |
| 285 | Build: release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{Schema: 1, Host: host, Completed: 1}}} |
| 286 | if err := release.WriteConsumed(b.optDir, release.NextConsumed(inB, release.TargetDecision{Version: "v-b", Reason: release.ReasonManual, Detail: "v-b"})); err != nil { |
| 287 | t.Fatal(err) |
| 288 | } |
| 289 | got = updatesReport(reg, b.optDir)[host] |
| 290 | if got.Account != "dee" || got.App != "blog" || got.Desired != "v-b" || got.Reason != "manual" || got.Detail != "v-b" { |
| 291 | t.Fatalf("B's entry after root decided for B = %+v; want B's decision", got) |
| 292 | } |
| 293 | if rep, err := updatesReader(reg, b.optDir)(host); err != nil || rep.Desired != "v-b" || rep.Reason != "manual" { |
| 294 | t.Fatalf("B's socket report after root decided for B = %+v, %v", rep, err) |
| 295 | } |
| 296 | } |
| 297 | |
| 298 | // R22: a consumed file from before root named the tenant is decided for no |
| 299 | // one: the entry names the route's tenant and carries the route's own major, |
| 300 | // and nothing that came from the file or the build state. |
| 301 | func TestAnUnattributedConsumedFileReportsNoDecision(t *testing.T) { |
| 302 | t.Parallel() |
| 303 | b, _, reg := testBuildStates(t) |
| 304 | const host = "a.example.com" |
| 305 | if err := release.WriteConsumed(b.optDir, release.Consumed{Schema: 1, Host: host, Desired: "v1", Reason: "pin", Detail: "v1", |
| 306 | Baking: "v2", Completed: 3, CompletedFailed: true}); err != nil { |
| 307 | t.Fatal(err) |
| 308 | } |
| 309 | two := 2 |
| 310 | if err := reg.SetUpdateSettings(host, release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "stable", Major: &two, InstallRequest: 7}.Column()); err != nil { |
| 311 | t.Fatal(err) |
| 312 | } |
| 313 | if err := reg.SetBuildState(host, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{Schema: 1, Host: host, Completed: 5}}.Column()); err != nil { |
| 314 | t.Fatal(err) |
| 315 | } |
| 316 | got := updatesReport(reg, b.optDir)[host] |
| 317 | if got.Account != release.OpsAccount || got.App != "km" || got.Major == nil || *got.Major != 2 { |
| 318 | t.Fatalf("entry = %+v; want tenant %s/km and the route's own major", got, release.OpsAccount) |
| 319 | } |
| 320 | if got.Desired != "" || got.Reason != "" || got.Detail != "" || got.Baking != "" || |
| 321 | got.Completed != 0 || got.CompletedFailed || got.Pending != 0 { |
| 322 | t.Fatalf("an unattributed file's decision was reported: %+v", got) |
| 323 | } |
| 324 | if rep, err := updatesReader(reg, b.optDir)(host); err != nil || rep.Desired != "" || rep.Reason != "" { |
| 325 | t.Fatalf("socket report = %+v, %v; want no decision from an unattributed file", rep, err) |
| 326 | } |
| 327 | // A route with default settings and only an unattributed file has |
| 328 | // nothing of its own to report, as before root decides for it. |
| 329 | if err := reg.SetUpdateSettings(host, ""); err != nil { |
| 330 | t.Fatal(err) |
| 331 | } |
| 332 | if got := updatesReport(reg, b.optDir); got != nil { |
| 333 | t.Fatalf("report = %+v; want none for a default route with no decision of its own", got) |
| 334 | } |
| 335 | } |
| 336 | |
| 337 | // R22: a file that names the route's tenant is reported in full; one that |
| 338 | // names the same account and another app, or another account and the same |
| 339 | // app, is not. |
| 340 | func TestAConsumedFileIsReportedOnlyForTheTenantItNames(t *testing.T) { |
| 341 | t.Parallel() |
| 342 | b, _, reg := testBuildStates(t) |
| 343 | const host = "a.example.com" |
| 344 | if err := reg.SetUpdateSettings(host, release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "stable", InstallRequest: 7}.Column()); err != nil { |
| 345 | t.Fatal(err) |
| 346 | } |
| 347 | write := func(account, app string) { |
| 348 | t.Helper() |
| 349 | if err := release.WriteConsumed(b.optDir, release.Consumed{Schema: 1, Host: host, Account: account, App: app, |
| 350 | Desired: "v1", Reason: "pin", Detail: "v1", Baking: "v2", Completed: 3, CompletedFailed: true}); err != nil { |
| 351 | t.Fatal(err) |
| 352 | } |
| 353 | } |
| 354 | write(release.OpsAccount, "km") |
| 355 | got := updatesReport(reg, b.optDir)[host] |
| 356 | if got.Desired != "v1" || got.Reason != "pin" || got.Detail != "v1" || got.Baking != "v2" || |
| 357 | got.Completed != 3 || !got.CompletedFailed || got.Pending != 7 { |
| 358 | t.Fatalf("matching file: entry = %+v; want the whole decision", got) |
| 359 | } |
| 360 | if rep, err := updatesReader(reg, b.optDir)(host); err != nil || rep.Desired != "v1" || rep.Reason != "pin" { |
| 361 | t.Fatalf("matching file: socket report = %+v, %v", rep, err) |
| 362 | } |
| 363 | for _, other := range [][2]string{{release.OpsAccount, "shop"}, {"cdf", "km"}, {"", "km"}, {release.OpsAccount, ""}} { |
| 364 | write(other[0], other[1]) |
| 365 | got := updatesReport(reg, b.optDir)[host] |
| 366 | if got.Desired != "" || got.Reason != "" || got.Detail != "" || got.Baking != "" || |
| 367 | got.Completed != 0 || got.CompletedFailed || got.Pending != 0 { |
| 368 | t.Fatalf("file for %v: entry = %+v; want no decision", other, got) |
| 369 | } |
| 370 | } |
| 371 | } |
| 372 | |
amadan