| 1 | package engines |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "sort" |
| 6 | "strings" |
| 7 | ) |
| 8 | |
| 9 | // The dedicated engine box and ingress (spec §5.9 "The engine box is |
| 10 | // dedicated"; §5.1 "The edge is the only ingress"). |
| 11 | // |
| 12 | // I1 keeps a declaring build from starting before the edge refuses the |
| 13 | // public, but the edge is not the only way to a process on its box. These |
| 14 | // rules close the others: |
| 15 | // |
| 16 | // - ports-excluded: a release declaring doc_engine and ports is refused at |
| 17 | // ship, and the box renders no ListenStream for an internal group. The |
| 18 | // capability pass removes an existing one whenever the group's mark |
| 19 | // closes (writeMark calls dropListeners), so it is gone before any fence |
| 20 | // of that write completes. |
| 21 | // - dedicated-launch-check: a declaring launch is refused on a box with a |
| 22 | // proxying row outside its group (the gate, before it writes or waits). |
| 23 | // - dedicated-edge-refusal: while an internal group is homed on the box, |
| 24 | // the edge gives every proxying row outside that group the internal |
| 25 | // refusal. |
| 26 | // - ingress-precondition: the operator's firewall admits connections from |
| 27 | // every source, public or private (the deployment's other boxes too), |
| 28 | // only on the edge's ports and the ports the operator granted. It is a |
| 29 | // world flag, not a box mechanism: the platform cannot see it. |
| 30 | // |
| 31 | // A proxying row is an instance or service row with a backend: the edge |
| 32 | // proxies it to a process. Static rows (and the status host, the tunnel and |
| 33 | // steering handlers, which the model does not have) are answered by the edge |
| 34 | // itself. Alias rows and path maps resolve to their final row, and the edge |
| 35 | // checks that one. |
| 36 | |
| 37 | // ---------- addresses ---------- |
| 38 | |
| 39 | // A TCP backend is on a box by its subnet, 10.0.<box>.<n>:<port>; a socket |
| 40 | // path or 127.0.0.1 is reachable on its own box only. |
| 41 | |
| 42 | // boxAddr is box b's address. |
| 43 | func boxAddr(b *box) string { return fmt.Sprintf("10.0.%d.1", b.ID) } |
| 44 | |
| 45 | // addrBox is the box an address backend is on, and its port; nil for a |
| 46 | // socket path, loopback, or an address on no box. |
| 47 | func (w *world) addrBox(backend string) (*box, string) { |
| 48 | i := strings.LastIndex(backend, ":") |
| 49 | if strings.HasPrefix(backend, "/") || i < 0 { |
| 50 | return nil, "" |
| 51 | } |
| 52 | for _, b := range w.boxes { |
| 53 | if strings.HasPrefix(backend[:i], fmt.Sprintf("10.0.%d.", b.ID)) { |
| 54 | return b, backend[i+1:] |
| 55 | } |
| 56 | } |
| 57 | return nil, "" |
| 58 | } |
| 59 | |
| 60 | // ownPort is the port process p binds on its own, if it does (OwnPort). |
| 61 | func ownPort(p *proc) string { return fmt.Sprint(20000 + p.ID) } |
| 62 | |
| 63 | // bindOwnPort is process p binding a port of its own on every interface: |
| 64 | // the unit template allows it and the capability validator leaves high |
| 65 | // ports to the app (spec §5.9, codex R12-1). |
| 66 | func (w *world) bindOwnPort(b *box, p *proc) { |
| 67 | p.OwnPort = true |
| 68 | w.logf("box%d: process %d (%s v%d) binds port %s on every interface", b.ID, p.ID, p.Group, p.Ver, ownPort(p)) |
| 69 | } |
| 70 | |
| 71 | // newest is group g's newest process on b, or nil. |
| 72 | func (w *world) newest(b *box, g string) *proc { |
| 73 | var p *proc |
| 74 | for _, x := range b.Procs { |
| 75 | if x.Group == g { |
| 76 | p = x |
| 77 | } |
| 78 | } |
| 79 | return p |
| 80 | } |
| 81 | |
| 82 | // ---------- service rows ---------- |
| 83 | |
| 84 | // proxying: the edge proxies the row to a process. |
| 85 | func proxying(r *row) bool { |
| 86 | return (r.Kind == "instance" || r.Kind == "service") && r.Backend != "" |
| 87 | } |
| 88 | |
| 89 | // addServiceRow is `carlos add` of a service row on box b whose upstream is |
| 90 | // backend: a socket path or address on b, or, from box 1, box 0's address. |
| 91 | // A service row is 0 by construction. |
| 92 | func (w *world) addServiceRow(b *box, host, backend string) { |
| 93 | b.Rows[host] = &row{Host: host, Kind: "service", Backend: backend, PathTo: map[string]string{}} |
| 94 | b.Mark[host] = markPublic |
| 95 | b.Wrote++ |
| 96 | w.logf("box%d: registry change %d: service row %s -> %s", b.ID, b.Wrote, host, backend) |
| 97 | } |
| 98 | |
| 99 | // deleteRow removes a service row. A request admitted through it keeps the |
| 100 | // proxy it holds (the fence's business, fence-box-wide). |
| 101 | func (w *world) deleteRow(b *box, host string) { |
| 102 | delete(b.Rows, host) |
| 103 | delete(b.Mark, host) |
| 104 | b.Wrote++ |
| 105 | w.logf("box%d: registry change %d: row %s deleted", b.ID, b.Wrote, host) |
| 106 | } |
| 107 | |
| 108 | // removeGroup is the operator removing instance route g from b, as the |
| 109 | // dedicated-box refusal's log line tells them to: its processes stop, its |
| 110 | // rows and the rows minted from it go, and so do its listeners. The app's |
| 111 | // staged versions stay on the box (/opt/carlos/versions is per app): any |
| 112 | // recorded under g pass to another instance of the app. |
| 113 | func (w *world) removeGroup(b *box, g string) { |
| 114 | others := []string{} |
| 115 | for _, h := range w.appGroups(b, g) { |
| 116 | if h != g { |
| 117 | others = append(others, h) |
| 118 | } |
| 119 | } |
| 120 | if len(others) > 0 { |
| 121 | for _, v := range b.Staged[g] { |
| 122 | b.Staged[others[0]] = append(b.Staged[others[0]], v) |
| 123 | } |
| 124 | } |
| 125 | w.stopGroup(b, g) |
| 126 | for _, h := range hostsOf(b) { |
| 127 | if r := b.Rows[h]; h == g || (r.Platform && r.Source == g) { |
| 128 | delete(b.Rows, h) |
| 129 | delete(b.Mark, h) |
| 130 | } |
| 131 | } |
| 132 | for port, lg := range b.Listen { |
| 133 | if lg == g { |
| 134 | delete(b.Listen, port) |
| 135 | } |
| 136 | } |
| 137 | delete(b.Live, g) |
| 138 | delete(b.Good, g) |
| 139 | delete(b.Staged, g) |
| 140 | delete(b.SetAt, g) |
| 141 | delete(b.PortGrant, g) |
| 142 | b.Wrote++ |
| 143 | w.logf("box%d: registry change %d: route %s removed with its minted rows", b.ID, b.Wrote, g) |
| 144 | } |
| 145 | |
| 146 | // ---------- the dedicated box ---------- |
| 147 | |
| 148 | // dedicatedLaunchCheck is the launch gate's check before it writes or waits |
| 149 | // for anything: a declaring launch in group g is refused while any proxying |
| 150 | // row of the box's registry belongs to another group, and each such row is |
| 151 | // named with the move or remove command. |
| 152 | func (w *world) dedicatedLaunchCheck(b *box, g string, ver int, path string) bool { |
| 153 | if !on("dedicated-launch-check") { |
| 154 | return true |
| 155 | } |
| 156 | var outside []string |
| 157 | for _, h := range hostsOf(b) { |
| 158 | if r := b.Rows[h]; proxying(r) && w.group(b, h) != g { |
| 159 | outside = append(outside, h) |
| 160 | } |
| 161 | } |
| 162 | for _, h := range outside { |
| 163 | w.logf("box%d: gate(%s) %s v%d: %s row %s proxies outside the group on the box that would home an engine; move or remove it", b.ID, path, g, ver, b.Rows[h].Kind, h) |
| 164 | } |
| 165 | if len(outside) > 0 { |
| 166 | w.logf("box%d: gate(%s) %s v%d: refused: the box is not dedicated", b.ID, path, g, ver) |
| 167 | return false |
| 168 | } |
| 169 | return true |
| 170 | } |
| 171 | |
| 172 | // edgeHomed is every internal group homed on b, as the edge's map has it: a |
| 173 | // group whose own route the edge serves 1 or NULL. The edge refuses NULL like |
| 174 | // 1 (classify-null-refused), and so does this rule, release or not: a gate |
| 175 | // writing 1 over a NULL group finds it fenced already and bumps no epoch, so |
| 176 | // a request admitted through a row beside a NULL route would outlive a |
| 177 | // declaring launch there (TestNullRouteHomesTheBox). The cost: a NULL route |
| 178 | // on a shared box (an upgrade that cannot read its live manifest, a |
| 179 | // provisioned route whose declaring first adoption the launch check refused) |
| 180 | // takes every proxying neighbour out of service until it is classified. |
| 181 | func (w *world) edgeHomed(b *box) map[string]bool { |
| 182 | homed := map[string]bool{} |
| 183 | for h, r := range b.EdgeRows { |
| 184 | if r.Kind != "instance" || r.Platform { |
| 185 | continue |
| 186 | } |
| 187 | if m := b.Serving[h]; m == markInternal || (m == markNull && on("classify-null-refused")) { |
| 188 | homed[h] = true |
| 189 | } |
| 190 | } |
| 191 | return homed |
| 192 | } |
| 193 | |
| 194 | // dedicatedRefusal is the edge's refusal of final, the row a public request |
| 195 | // resolved to: on a box that homes an internal group, a proxying row of any |
| 196 | // other group gets the internal refusal. It returns why, or "" when the row |
| 197 | // is served. |
| 198 | func (w *world) dedicatedRefusal(b *box, final row) string { |
| 199 | if !on("dedicated-edge-refusal") || !proxying(&final) { |
| 200 | return "" |
| 201 | } |
| 202 | homed := w.edgeHomed(b) |
| 203 | if len(homed) == 0 || homed[edgeGroup(b, final.Host)] { |
| 204 | return "" |
| 205 | } |
| 206 | return fmt.Sprintf("%s row %s proxies outside %v, the internal group homed on the box; move or remove it", final.Kind, final.Host, sortedKeys(homed)) |
| 207 | } |
| 208 | |
| 209 | func sortedKeys(m map[string]bool) []string { |
| 210 | out := make([]string, 0, len(m)) |
| 211 | for k := range m { |
| 212 | out = append(out, k) |
| 213 | } |
| 214 | sort.Strings(out) |
| 215 | return out |
| 216 | } |
| 217 | |
| 218 | // ---------- ports ---------- |
| 219 | |
| 220 | // ship is `carlos ship` of build ver's release for group g, its capability |
| 221 | // block declaring ports or not, then its adoption on b. A ports release |
| 222 | // records the operator's grant (a port) and the capability pass runs. |
| 223 | func (w *world) ship(b *box, g string, ver int, ports bool) bool { |
| 224 | if ports && w.builds[ver].Declares && on("ports-excluded") { |
| 225 | w.logf("ship %s v%d: refused: the release declares doc_engine and ports together", g, ver) |
| 226 | return false |
| 227 | } |
| 228 | if !w.adopt(b, g, ver, nil) { |
| 229 | return false |
| 230 | } |
| 231 | if ports && b.PortGrant[g] == "" { |
| 232 | w.nextID++ |
| 233 | b.PortGrant[g] = fmt.Sprint(40000 + w.nextID) |
| 234 | w.logf("box%d: %s holds a ports grant: port %s", b.ID, g, b.PortGrant[g]) |
| 235 | } |
| 236 | w.capsPass(b) |
| 237 | return true |
| 238 | } |
| 239 | |
| 240 | // capsPass is the agent's capability pass: it renders each granted port's |
| 241 | // ListenStream, except for a route of an internal group, whose listener it |
| 242 | // removes. It fails closed: the route's own mark or any row of its group |
| 243 | // being 1 or NULL is enough. |
| 244 | func (w *world) capsPass(b *box) { |
| 245 | gs := make([]string, 0, len(b.PortGrant)) |
| 246 | for g := range b.PortGrant { |
| 247 | gs = append(gs, g) |
| 248 | } |
| 249 | sort.Strings(gs) |
| 250 | for _, g := range gs { |
| 251 | if on("ports-excluded") && (b.Mark[g] != markPublic || w.closed(b, g)) { |
| 252 | w.dropListeners(b, g) |
| 253 | continue |
| 254 | } |
| 255 | if port := b.PortGrant[g]; b.Listen[port] != g { |
| 256 | b.Listen[port] = g |
| 257 | w.logf("box%d: capability pass renders ListenStream=%s for %s", b.ID, port, g) |
| 258 | } |
| 259 | } |
| 260 | } |
| 261 | |
| 262 | // dropListeners removes group g's ListenStream sockets, stopping their |
| 263 | // socket units, and logs the grant it is ignoring. writeMark calls it on |
| 264 | // every write of 1 or NULL. |
| 265 | func (w *world) dropListeners(b *box, g string) { |
| 266 | if !on("ports-excluded") { |
| 267 | return |
| 268 | } |
| 269 | ports := make([]string, 0, len(b.Listen)) |
| 270 | for port, lg := range b.Listen { |
| 271 | if lg == g { |
| 272 | ports = append(ports, port) |
| 273 | } |
| 274 | } |
| 275 | sort.Strings(ports) |
| 276 | for _, port := range ports { |
| 277 | delete(b.Listen, port) |
| 278 | w.logf("box%d: %s is internal: ListenStream=%s removed, its ports grant ignored (revoke it with carlos caps)", b.ID, g, port) |
| 279 | } |
| 280 | } |
| 281 | |
| 282 | // ---------- ingress ---------- |
| 283 | |
| 284 | // directConnect is a connection to port on box b that does not go through |
| 285 | // b's edge: from the internet, or from another box (a service row there |
| 286 | // aimed at b's address). A ports listener takes it whatever the firewall |
| 287 | // flag (the operator granted that port, so the firewall admits it); a stopped |
| 288 | // unit is socket-activated from its live link. Otherwise it reaches a process |
| 289 | // only if the ingress precondition is violated: one bound to the port on its |
| 290 | // own, or a TCP route's backend. |
| 291 | func (w *world) directConnect(b *box, port string, public bool) *proc { |
| 292 | return w.connect(b, port, public, true) |
| 293 | } |
| 294 | |
| 295 | // connect is directConnect, or with firewalled false the box's own traffic |
| 296 | // (a local row's proxy to an address on its own box), which no firewall sees. |
| 297 | func (w *world) connect(b *box, port string, public, firewalled bool) *proc { |
| 298 | src := "private" |
| 299 | if public { |
| 300 | src = "public" |
| 301 | } |
| 302 | if g, ok := b.Listen[port]; ok { |
| 303 | w.logf("box%d: %s connection to port %s: the ports listener of %s", b.ID, src, port, g) |
| 304 | w.socketActivate(b, g) |
| 305 | return w.reach(b, w.newest(b, g), public, "the ports listener on "+port) |
| 306 | } |
| 307 | if firewalled && on("ingress-precondition") { |
| 308 | w.logf("box%d: %s connection to port %s: refused by the firewall (the edge's ports only)", b.ID, src, port) |
| 309 | return nil |
| 310 | } |
| 311 | return w.reach(b, w.portProc(b, port), public, "port "+port) |
| 312 | } |
| 313 | |
| 314 | // portProc is the process listening on port on b: one bound to it on its |
| 315 | // own, else the newest process of the TCP route whose backend it is. |
| 316 | func (w *world) portProc(b *box, port string) *proc { |
| 317 | for i := len(b.Procs) - 1; i >= 0; i-- { |
| 318 | if p := b.Procs[i]; p.OwnPort && ownPort(p) == port { |
| 319 | return p |
| 320 | } |
| 321 | } |
| 322 | for _, h := range hostsOf(b) { |
| 323 | r := b.Rows[h] |
| 324 | if r.Kind != "instance" || r.Platform { |
| 325 | continue |
| 326 | } |
| 327 | if tb, pt := w.addrBox(r.Backend); tb == b && pt == port { |
| 328 | return w.newest(b, w.group(b, h)) |
| 329 | } |
| 330 | } |
| 331 | return nil |
| 332 | } |
| 333 | |
| 334 | // reach records a connection past the edge reaching process p. INV-P, as in |
| 335 | // deliver: a public connection never reaches a declaring build, nor a group |
| 336 | // holding a loaded doc-engine grant. |
| 337 | func (w *world) reach(b *box, p *proc, public bool, how string) *proc { |
| 338 | if p == nil { |
| 339 | w.logf("box%d: %s: nothing listens", b.ID, how) |
| 340 | return nil |
| 341 | } |
| 342 | w.logf("box%d: %s reached process %d (%s v%d) past the edge", b.ID, how, p.ID, p.Group, p.Ver) |
| 343 | w.events["delivered"]++ |
| 344 | // a ports listener exists only because the group is public: a path the |
| 345 | // upgrade's 0 opens; any other way past the edge is the ingress rule's |
| 346 | listener := strings.HasPrefix(how, "the ports listener") |
| 347 | if public && w.builds[p.Ver].Declares { |
| 348 | w.failProc(b, p, "INV-P", listener, "box%d: a public connection through %s reached process %d of %s running v%d, which declares doc_engine, past the edge", b.ID, how, p.ID, p.Group, p.Ver) |
| 349 | } else if g := w.grantLoaded(b, p.Group); public && g != "" { |
| 350 | w.failProc(b, p, "INV-P", listener, "box%d: a public connection through %s reached process %d of %s running v%d, whose group holds the doc-engine grant (on %s), past the edge", b.ID, how, p.ID, p.Group, p.Ver, g) |
| 351 | } |
| 352 | return p |
| 353 | } |
| 354 | |
| 355 | // deliverDirect is the proxy of request r on b whose backend is an address |
| 356 | // no instance row of b carries: on another box, a direct connection there |
| 357 | // (the firewall decides); on b itself, the process bound to that port. False |
| 358 | // when the backend is no such address. |
| 359 | func (w *world) deliverDirect(b *box, r *req) bool { |
| 360 | tb, port := w.addrBox(r.Backend) |
| 361 | if tb == nil { |
| 362 | return false |
| 363 | } |
| 364 | w.logf("box%d: request %d proxies to %s, box%d's address", b.ID, r.ID, r.Backend, tb.ID) |
| 365 | if p := w.connect(tb, port, r.Public, tb != b); p != nil { |
| 366 | r.Reached = p.ID |
| 367 | } |
| 368 | return true |
| 369 | } |
| 370 | |
| 371 | // ---------- INV-AV ---------- |
| 372 | |
| 373 | // A proxying row outside the internal group homed on its box is refused by |
| 374 | // the dedicated-box rule on purpose (spec §5.9): INV-AV does not count it. |
| 375 | // The row's true group decides, not the edge's resolution. |
| 376 | func init() { |
| 377 | avExemptions = append(avExemptions, func(w *world, b *box, host string) string { |
| 378 | r := b.Rows[host] |
| 379 | if r == nil || !proxying(r) { |
| 380 | return "" |
| 381 | } |
| 382 | homed := w.edgeHomed(b) |
| 383 | if len(homed) == 0 || homed[truthGroup(b, host)] { |
| 384 | return "" |
| 385 | } |
| 386 | return fmt.Sprintf("a proxying row outside %v, the internal group homed on the box: refused by design (§5.9)", sortedKeys(homed)) |
| 387 | }) |
| 388 | } |
| 389 | |
| 390 | // ---------- the random walk ---------- |
| 391 | |
| 392 | // addressablePorts is every port on b a direct connection can name: TCP |
| 393 | // routes' backends, ports processes bound on their own, ports listeners. |
| 394 | func (w *world) addressablePorts(b *box) []string { |
| 395 | set := map[string]bool{} |
| 396 | for _, h := range hostsOf(b) { |
| 397 | if r := b.Rows[h]; r.Kind == "instance" && !r.Platform { |
| 398 | if tb, port := w.addrBox(r.Backend); tb == b { |
| 399 | set[port] = true |
| 400 | } |
| 401 | } |
| 402 | } |
| 403 | for _, p := range b.Procs { |
| 404 | if p.OwnPort { |
| 405 | set[ownPort(p)] = true |
| 406 | } |
| 407 | } |
| 408 | for port := range b.Listen { |
| 409 | set[port] = true |
| 410 | } |
| 411 | return sortedKeys(set) |
| 412 | } |
| 413 | |
| 414 | // localBackends is every upstream a local service row on b can name: its |
| 415 | // instance routes' backends and the ports on b's own address. |
| 416 | func (w *world) localBackends(b *box) []string { |
| 417 | set := map[string]bool{"127.0.0.1:1": true} // nothing listens |
| 418 | for _, h := range hostsOf(b) { |
| 419 | if r := b.Rows[h]; r.Kind == "instance" && !r.Platform && r.Backend != "" { |
| 420 | set[r.Backend] = true |
| 421 | } |
| 422 | } |
| 423 | for _, port := range w.addressablePorts(b) { |
| 424 | set[boxAddr(b)+":"+port] = true |
| 425 | } |
| 426 | return sortedKeys(set) |
| 427 | } |
| 428 | |
| 429 | func init() { |
| 430 | actions = append(actions, |
| 431 | action{name: "add-service-row", weight: 2, run: func(w *world) bool { |
| 432 | b := w.pickBox() |
| 433 | if len(b.Rows) >= maxRows { |
| 434 | return false |
| 435 | } |
| 436 | w.nextID++ |
| 437 | host := fmt.Sprintf("svc%d.example", w.nextID) |
| 438 | if b.ID == 1 && w.rng.Intn(2) == 0 { |
| 439 | // aimed at box 0's address (codex R13-1) |
| 440 | ports := append(w.addressablePorts(w.boxes[0]), "1") |
| 441 | w.addServiceRow(b, host, boxAddr(w.boxes[0])+":"+ports[w.rng.Intn(len(ports))]) |
| 442 | return true |
| 443 | } |
| 444 | bs := w.localBackends(b) |
| 445 | w.addServiceRow(b, host, bs[w.rng.Intn(len(bs))]) |
| 446 | return true |
| 447 | }}, |
| 448 | action{name: "delete-service-row", weight: 2, run: func(w *world) bool { |
| 449 | type at struct { |
| 450 | b *box |
| 451 | host string |
| 452 | } |
| 453 | var rows []at |
| 454 | for _, b := range w.boxes { |
| 455 | for _, h := range hostsOf(b) { |
| 456 | if b.Rows[h].Kind == "service" { |
| 457 | rows = append(rows, at{b, h}) |
| 458 | } |
| 459 | } |
| 460 | } |
| 461 | if len(rows) == 0 { |
| 462 | return false |
| 463 | } |
| 464 | x := rows[w.rng.Intn(len(rows))] |
| 465 | w.deleteRow(x.b, x.host) |
| 466 | return true |
| 467 | }}, |
| 468 | action{name: "own-port", weight: 1, run: func(w *world) bool { |
| 469 | b := w.pickBox() |
| 470 | if len(b.Procs) == 0 { |
| 471 | return false |
| 472 | } |
| 473 | w.bindOwnPort(b, b.Procs[w.rng.Intn(len(b.Procs))]) |
| 474 | return true |
| 475 | }}, |
| 476 | action{name: "direct-connect", weight: 2, run: func(w *world) bool { |
| 477 | b := w.pickBox() |
| 478 | ports := append(w.addressablePorts(b), "1") |
| 479 | w.directConnect(b, ports[w.rng.Intn(len(ports))], true) |
| 480 | return true |
| 481 | }}, |
| 482 | action{name: "ship-ports", weight: 1, run: func(w *world) bool { |
| 483 | b := w.pickBox() |
| 484 | g := w.pickGroup(b) |
| 485 | if g == "" { |
| 486 | return false |
| 487 | } |
| 488 | w.ship(b, g, w.pickVer(), true) |
| 489 | return true |
| 490 | }}, |
| 491 | action{name: "caps-pass", weight: 1, run: func(w *world) bool { w.capsPass(w.pickBox()); return true }}, |
| 492 | action{name: "remove-route", weight: 1, run: func(w *world) bool { |
| 493 | // an instance route the walk added (the starting eng and shop stay) |
| 494 | b := w.pickBox() |
| 495 | var gs []string |
| 496 | for _, g := range w.groups(b) { |
| 497 | if g != eng && g != "shop" { |
| 498 | gs = append(gs, g) |
| 499 | } |
| 500 | } |
| 501 | if len(gs) == 0 { |
| 502 | return false |
| 503 | } |
| 504 | w.removeGroup(b, gs[w.rng.Intn(len(gs))]) |
| 505 | return true |
| 506 | }}, |
| 507 | ) |
| 508 | } |
| 509 | |
amadan