| 1 | package hostlife |
| 2 | |
| 3 | // ---------- the application ---------- |
| 4 | |
| 5 | // traffic: every running process of the host does its own work, whether or |
| 6 | // not a request arrived: background workers and descendants included. |
| 7 | func (w *world) traffic(b *box) { |
| 8 | if b.Destroyed || b.Frozen { |
| 9 | return |
| 10 | } |
| 11 | for _, p := range b.Procs { |
| 12 | if w.rng.Intn(3) != 0 { |
| 13 | continue |
| 14 | } |
| 15 | w.nextW++ |
| 16 | wr := w.nextW |
| 17 | if b.DB != nil && b.DB.Inc == p.Inc { |
| 18 | w.commit(b.DB, wr) |
| 19 | } else { |
| 20 | w.txs[wr] = tx{W: wr} // into a purged or foreign database: nowhere |
| 21 | } |
| 22 | // acknowledged to the client or the outside world either way: a |
| 23 | // write into a purged or foreign database is a lost write. The |
| 24 | // acknowledgement is the process's session's (rev 7.13, R12-1): the |
| 25 | // database under it may be gone (seed 899 / 3,000) |
| 26 | w.acks = append(w.acks, ack{W: wr, Inc: p.Inc, Box: b.ID, Session: p.Session, Gen: p.Gen, At: w.now, Kind: "write"}) |
| 27 | w.writeInc[wr] = p.Inc |
| 28 | if p.Main && w.rng.Intn(4) == 0 && b.Row != nil && b.Row.Inc == p.Inc { |
| 29 | w.nextW++ |
| 30 | b.Timers = append(b.Timers, w.nextW) |
| 31 | w.acks = append(w.acks, ack{W: w.nextW, Inc: p.Inc, Box: b.ID, Session: p.Session, Gen: p.Gen, At: w.now, Kind: "timer"}) |
| 32 | } |
| 33 | } |
| 34 | } |
| 35 | |
| 36 | // idleStop: the activator's ordinary idle stop (main only; a descendant |
| 37 | // keeps running, as today). |
| 38 | func (w *world) idleStop(b *box) { |
| 39 | if !w.agentUp(b) || !hasMain(b) { |
| 40 | return |
| 41 | } |
| 42 | w.replicate(b) |
| 43 | // the unit stop kills the cgroup: the main process and every descendant |
| 44 | // placed in it; an escaped descendant survives, and while it lives the |
| 45 | // session cannot certify (rev 7.10) |
| 46 | var keep []*proc |
| 47 | for _, p := range b.Procs { |
| 48 | if !p.Main && !p.Strict { |
| 49 | keep = append(keep, p) |
| 50 | } |
| 51 | } |
| 52 | b.Procs = keep |
| 53 | if b.open != nil { |
| 54 | // the session's start is still finishing its PUTs: the agent's |
| 55 | // replicator opens when they land and ships what the app wrote, |
| 56 | // then closes (rev 7.11 §7.14); no certificate now |
| 57 | w.stats["idle stop before replication opened"]++ |
| 58 | return |
| 59 | } |
| 60 | // the clean close (claim rev 6.9 §6.4a): drain, then the certificate |
| 61 | if b.DB != nil { |
| 62 | b.DB.Closing = true |
| 63 | w.closeStep(b) |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | // wake: a request reaches this box's edge for the host. |
| 68 | func (w *world) wake(b *box) { |
| 69 | if !w.agentUp(b) || b.Row == nil || b.Row.Status != "ready" { |
| 70 | return |
| 71 | } |
| 72 | // the edge routes on the row (edge.go:595): a ready row receives the |
| 73 | // public request whether or not the instance already runs. INV-5: no |
| 74 | // public delivery to a landing target before the commit (R3-5) |
| 75 | if c := w.claim; c != nil && c.Op.Kind == "landing" && !c.Op.Committed && c.Op.Tgt == b.ID { |
| 76 | w.violate("INV-5: a public request reached the landing target, box %d, before the commit of %s", b.ID, c.Op.ID) |
| 77 | return |
| 78 | } |
| 79 | if hasMain(b) { |
| 80 | return |
| 81 | } |
| 82 | if b.Old { |
| 83 | if w.rec == nil { |
| 84 | return |
| 85 | } |
| 86 | c := &claim{Holder: &holder{Inc: w.rec.Inc, App: w.rec.App, Legacy: true}} |
| 87 | w.spawn(b, c, "", false) |
| 88 | return |
| 89 | } |
| 90 | c, ok := w.admissionClaim(b) |
| 91 | if !ok || !w.admits(b, c, "") { |
| 92 | return |
| 93 | } |
| 94 | if !w.completedHandover(b, c) { |
| 95 | return // N6: the first serving wake after a completion hands over from the probe's receipt |
| 96 | } |
| 97 | if b.DB == nil && on("quiet-before-ack") && w.now < c.Quiet[b.Region] { |
| 98 | return // a first install into this region waits out quiet_after (claim §6.2a) |
| 99 | } |
| 100 | w.spawn(b, c, c.Holder.Gen, false) |
| 101 | } |
| 102 | |
| 103 | // ---------- reconcile ---------- |
| 104 | |
| 105 | func (w *world) retireRow(b *box) { |
| 106 | if b.Row == nil { |
| 107 | return |
| 108 | } |
| 109 | if !b.Old && !w.evidence(b, "retire") { |
| 110 | return |
| 111 | } |
| 112 | if b.Old { |
| 113 | var keep []*proc |
| 114 | for _, p := range b.Procs { |
| 115 | if !p.Main { |
| 116 | keep = append(keep, p) |
| 117 | } |
| 118 | } |
| 119 | b.Procs = keep |
| 120 | } |
| 121 | b.Row = nil |
| 122 | b.Config = 0 // reapConfigEnv |
| 123 | // the schedule converger prunes timers of a host with no row |
| 124 | b.Timers, b.History, b.Attempt = nil, nil, 0 |
| 125 | } |
| 126 | |
| 127 | func (w *world) ownsLocalObligation(c *claim, b *box) bool { |
| 128 | for _, o := range c.Obl { |
| 129 | if o.Kind == "park-local" && o.Box == b.ID { |
| 130 | return true |
| 131 | } |
| 132 | } |
| 133 | return false |
| 134 | } |
| 135 | |
| 136 | func (w *world) reconcile(b *box) { |
| 137 | if !w.agentUp(b) { |
| 138 | return |
| 139 | } |
| 140 | if b.Old { |
| 141 | if w.rec != nil && w.rec.Region == b.Region { |
| 142 | if b.Row == nil || b.Row.Inc != w.rec.Inc { |
| 143 | b.Row = &row{Inc: w.rec.Inc, App: w.rec.App, Status: "ready"} |
| 144 | } |
| 145 | } else if b.Row != nil { |
| 146 | w.retireRow(b) |
| 147 | } |
| 148 | return |
| 149 | } |
| 150 | c, ok := w.admissionClaim(b) // reconcile's keep-or-create is an admission decision (C4) |
| 151 | if !ok { |
| 152 | return |
| 153 | } |
| 154 | if c == nil { |
| 155 | if w.claim == nil { |
| 156 | w.adopt() |
| 157 | } |
| 158 | return |
| 159 | } |
| 160 | w.reconcileClaimed(b, c) |
| 161 | } |
| 162 | |
| 163 | // adopt is the one adoption routine every actor uses (claim §9): a |
| 164 | // create-only claim for an unclaimed legacy record, with discovery. |
| 165 | func (w *world) adopt() bool { |
| 166 | if w.claim != nil || w.rec == nil { |
| 167 | return false |
| 168 | } |
| 169 | eb := w.adoptionOwner(w.rec.Region) // exactly one proven owner (claim P5-D2) |
| 170 | if eb == nil { |
| 171 | return false |
| 172 | } |
| 173 | // adoption writes holder.stamp (claim rev 6.5 §2). The pre-claim writer |
| 174 | // is the legacy identity (claim rev 6.24, R16-3): session 0 with |
| 175 | // generation "" (read as @root), the pre-release process on the |
| 176 | // executing box its writer_box; the first cold admission's writes_lost |
| 177 | // names it (seed 94288 / 100,000 named box 0 and left the legacy writes |
| 178 | // undeclared), and a lost-target return declares it in |
| 179 | // additional_writes_lost. Rev 7.13 to 7.16 gave it session 1, which hid |
| 180 | // the session-zero case from E |
| 181 | n := &claim{Ver: 1, Holder: &holder{Inc: w.rec.Inc, App: w.rec.App, Legacy: true, Location: locationFor(w.rec.Inc, true), Stamp: 1, Session: 0, WriterBox: eb.ID}, |
| 182 | Op: opState{Kind: "serving", Exec: eb.ID, Parts: []int{eb.ID}}, Regions: []int{w.rec.Region}, Boxes: []int{eb.ID}, Complete: false} |
| 183 | if w.stampBox == nil { |
| 184 | w.stampBox = map[int]int{} |
| 185 | } |
| 186 | w.stampBox[1] = eb.ID // the adoption's stamp names the serving box's installation (claim rev 6.5 §2) |
| 187 | if on("legacy-discovery") { |
| 188 | // every other region parks whatever it holds: it cannot be the |
| 189 | // live lineage, because the record names another region |
| 190 | for r := 0; r < nRegions; r++ { |
| 191 | if r == w.rec.Region { |
| 192 | continue |
| 193 | } |
| 194 | for _, p := range []string{"instances/" + host + "/", "moves/" + host + "/"} { |
| 195 | n.Obl = append(n.Obl, obligation{ID: w.newID("ob"), Op: "adopt", Kind: "park-replica", Region: r, Box: -1, Prefix: p, Ret: "legacy"}) |
| 196 | } |
| 197 | } |
| 198 | // and local state on every box but the record region's serving box |
| 199 | // (claim rev 4 §9, the J2 N5 note): it cannot be the live instance's |
| 200 | for _, b := range w.liveBoxes() { |
| 201 | if b.ID != eb.ID { |
| 202 | n.Obl = append(n.Obl, obligation{ID: w.newID("ob"), Op: "adopt", Kind: "park-local", Box: b.ID, Region: -1, Ret: "legacy"}) |
| 203 | } |
| 204 | } |
| 205 | } |
| 206 | w.claim = n |
| 207 | w.logf("adopted legacy host inc %d on box %d", w.rec.Inc, eb.ID) |
| 208 | return true |
| 209 | } |
| 210 | |
| 211 | func (w *world) reconcileClaimed(b *box, c *claim) { |
| 212 | h := c.Holder |
| 213 | if h == nil || w.rec == nil || w.rec.Inc != h.Inc { |
| 214 | if b.Row != nil && !w.ownsLocalObligation(c, b) { |
| 215 | w.retireRow(b) |
| 216 | } |
| 217 | return |
| 218 | } |
| 219 | if b.Row != nil && b.Row.Inc != h.Inc { |
| 220 | w.retireRow(b) // an orphan row (claim C1) |
| 221 | return |
| 222 | } |
| 223 | switch c.Op.Kind { |
| 224 | case "serving", "creating", "pushing": |
| 225 | if c.Op.Exec == b.ID { |
| 226 | if b.Row != nil && b.Row.Status != "ready" && c.Op.Kind == "serving" && on("stale-hold-release") && !w.completionHolds(c, b) { |
| 227 | // a hold whose operation has ended (the claim is serving here) |
| 228 | b.Row.Status, b.Row.HoldOp = "ready", "" |
| 229 | } |
| 230 | if b.Row == nil && c.Op.Kind == "serving" { |
| 231 | b.Row = &row{Inc: h.Inc, App: h.App, Status: "ready"} |
| 232 | w.initRecord(b, c) // the local record from holder.stamp, on this fresh read (claim rev 6.5 §6.4) |
| 233 | } |
| 234 | } else if w.catchUpHold(b, c) { |
| 235 | // a delayed source's row, held for the completed op until its |
| 236 | // park-local retires it (finding 3, ruling 85) |
| 237 | } else if b.Row != nil && !(on("participant-rows") && w.ownsLocalObligation(c, b)) { |
| 238 | w.retireRow(b) |
| 239 | } |
| 240 | default: |
| 241 | if c.Op.Kind == "relocating" && b.ID == c.Op.Src && b.Row != nil && b.Row.Status == "ready" && c.Op.Release != nil { |
| 242 | // §5.3, L12 (rule reloc-row-cases): the former executor's row is |
| 243 | // held once its admission fence closed, never ready, until its |
| 244 | // park-local retires it. The mutation reopens it as a stale hold |
| 245 | if on("reloc-row-cases") { |
| 246 | b.Row.Status, b.Row.HoldOp = "relocating", c.Op.ID |
| 247 | } |
| 248 | } |
| 249 | if w.isParticipant(c, b) { |
| 250 | if !on("participant-rows") && w.rec.Region != b.Region && b.Row != nil { |
| 251 | w.retireRow(b) // region mismatch retires a participant's row |
| 252 | } |
| 253 | return |
| 254 | } |
| 255 | if b.Row != nil && !w.ownsLocalObligation(c, b) { |
| 256 | w.retireRow(b) |
| 257 | } |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | // ---------- background passes with work in flight (J-N8) ---------- |
| 262 | |
| 263 | func (w *world) passPlan(b *box, kind string) { |
| 264 | if !w.agentUp(b) || b.Row == nil || b.Row.Status != "ready" { |
| 265 | return |
| 266 | } |
| 267 | if kind == "config" && !b.HasKey { |
| 268 | return |
| 269 | } |
| 270 | b.plans = append(b.plans, &plan{Kind: kind, Inc: b.Row.Inc, App: b.Row.App, Ver: w.cfgVer[b.Row.App], At: w.now + w.pauseLen(), Stamp: b.stampOf(b.Row.Inc)}) |
| 271 | } |
| 272 | |
| 273 | // passExec runs one independent process's write. It needs the box running, |
| 274 | // not the agent. Under the host guard (an exclusive flock shared by the |
| 275 | // agent and every root command, claim rev 4 §6.4) it reads the claim |
| 276 | // fresh and refuses if the incarnation, the operation or the config |
| 277 | // identity it selected has moved on. The mutation keeps an in-process lock |
| 278 | // only: the process acts on what it validated before it paused. |
| 279 | func (w *world) passExec(b *box) { |
| 280 | if b.Destroyed || b.Frozen || len(b.plans) == 0 { |
| 281 | return |
| 282 | } |
| 283 | i := w.rng.Intn(len(b.plans)) |
| 284 | p := b.plans[i] |
| 285 | if p.At > w.now { |
| 286 | return |
| 287 | } |
| 288 | b.plans = append(b.plans[:i:i], b.plans[i+1:]...) |
| 289 | if on("hostguard") { |
| 290 | if b.Partitioned { |
| 291 | return |
| 292 | } |
| 293 | c := w.claim |
| 294 | if c == nil || c.Holder == nil || c.Holder.Inc != p.Inc || b.Row == nil || b.Row.Inc != p.Inc { |
| 295 | return |
| 296 | } |
| 297 | if p.Kind == "config" && p.Ver != w.cfgVer[p.App] { |
| 298 | return // the selected bundle is no longer the held one |
| 299 | } |
| 300 | } |
| 301 | // the local record (claim rev 6.5 §6.4): the writer carries the stamp |
| 302 | // it selected under, admitted only if it is still the active one |
| 303 | if !b.admitsStamp(p.Inc, p.Stamp) { |
| 304 | return |
| 305 | } |
| 306 | if p.Stamp != b.stampOf(p.Inc) { |
| 307 | // the oracle: a writer of a retired installation wrote host state |
| 308 | w.violate("INV-6: a %s pass of a retired installation (stamp %d, active %d) wrote host state for inc %d on box %d", p.Kind, p.Stamp, b.stampOf(p.Inc), p.Inc, b.ID) |
| 309 | } |
| 310 | switch p.Kind { |
| 311 | case "config": |
| 312 | b.Config, b.CfgVer = p.Inc, p.Ver |
| 313 | case "adopt": |
| 314 | b.Live, b.LiveInc = p.App, p.Inc |
| 315 | } |
| 316 | } |
| 317 | |
| 318 | // ---------- timers ---------- |
| 319 | |
| 320 | func (w *world) runner(b *box) { |
| 321 | if !w.agentUp(b) || b.Row == nil { |
| 322 | return |
| 323 | } |
| 324 | if len(b.Timers) > 0 { |
| 325 | if b.Row.Status == "ready" && hasMain(b) { |
| 326 | id := b.Timers[0] |
| 327 | b.Timers = b.Timers[1:] |
| 328 | b.History = append(b.History, id) |
| 329 | w.delivered[id] = true |
| 330 | } else if b.Row.Status != "ready" && !on("timer-freeze") { |
| 331 | b.Attempt++ // a refused delivery still advances bookkeeping |
| 332 | } |
| 333 | } |
| 334 | if w.rng.Intn(4) == 0 { |
| 335 | b.History = nil // the runner's own daily prune (runner.go:260) |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | // ---------- self-fencing (claim §7.1) ---------- |
| 340 | |
| 341 | func (w *world) watchdog(b *box) { |
| 342 | if b.Destroyed || b.Frozen { |
| 343 | return |
| 344 | } |
| 345 | if !on("watchdog-independent") && !w.agentUp(b) { |
| 346 | return |
| 347 | } |
| 348 | if b.ExecUntil > 0 && w.localNow(b) > b.ExecUntil && len(b.Procs) > 0 { |
| 349 | b.Procs = nil |
| 350 | w.logf("box %d self-fenced (exec lease expired)", b.ID) |
| 351 | } |
| 352 | } |
| 353 | |
| 354 | // ---------- faults ---------- |
| 355 | |
| 356 | func (w *world) fault() { |
| 357 | if len(w.pending) > 0 && w.rng.Intn(3) == 0 { |
| 358 | i := w.rng.Intn(len(w.pending)) |
| 359 | r := w.pending[i] |
| 360 | if r.beyond && w.fail["recreate"] { |
| 361 | return // the biased schedule keeps it for its worst moment |
| 362 | } |
| 363 | w.pending = append(w.pending[:i:i], w.pending[i+1:]...) |
| 364 | w.complete(r) |
| 365 | return |
| 366 | } |
| 367 | bs := w.liveBoxes() |
| 368 | b := bs[w.rng.Intn(len(bs))] |
| 369 | switch w.rng.Intn(9) { |
| 370 | case 0: |
| 371 | if !b.AgentDown { |
| 372 | b.AgentDown = true |
| 373 | var keep []*proc |
| 374 | for _, p := range b.Procs { |
| 375 | if b.Nested { |
| 376 | keep = append(keep, p) |
| 377 | } |
| 378 | } |
| 379 | b.Procs = keep |
| 380 | b.forget() |
| 381 | w.logf("box %d agent crashed", b.ID) |
| 382 | // a crash that takes the disk with it (an instance replaced, a |
| 383 | // volume gone): the writer's uploads may still be in flight, |
| 384 | // which is the writer-epoch hazard (claim rev 6.6 §6.4a, R7-1) |
| 385 | // in every operation that has this box write (R8-2), not only |
| 386 | // while serving |
| 387 | if c := w.claim; w.fail["disk"] && c != nil && c.Holder != nil && c.Op.Exec == b.ID && b.DB != nil && w.rng.Intn(3) == 0 { |
| 388 | // the fault removes physical state only (R10-6): the |
| 389 | // acknowledged, unuploaded writes are declared lost by the |
| 390 | // next admission's writes_lost record, never exempted here |
| 391 | b.DB = nil |
| 392 | b.Procs = nil // nothing survives the disk: a descendant with no database would acknowledge writes into nowhere |
| 393 | w.stats["disk-lost with the crash"]++ |
| 394 | w.logf("box %d lost its disk with the crash", b.ID) |
| 395 | } |
| 396 | } else { |
| 397 | b.AgentDown = false |
| 398 | } |
| 399 | case 1: |
| 400 | b.Procs = nil |
| 401 | b.plans = nil // a reboot ends every process |
| 402 | b.AgentDown, b.Frozen, b.AgentPaused = false, false, false |
| 403 | b.forget() |
| 404 | if w.rng.Intn(6) == 0 { |
| 405 | // the purge of the local database the next admitted writer must |
| 406 | // make returns an error for a while (claim rev 6.9): the writer |
| 407 | // does not start |
| 408 | b.PurgeFails = !b.PurgeFails |
| 409 | if b.PurgeFails { |
| 410 | w.stats["purge failures injected"]++ |
| 411 | } |
| 412 | } |
| 413 | if w.rng.Intn(8) == 0 && len(b.Stamps) > 0 { |
| 414 | // the local stamp record is lost with the reboot: the box |
| 415 | // refuses every writer until it re-initializes from a fresh |
| 416 | // claim read naming it (claim rev 6.5 §6.4; rev 7.7, R6-4) |
| 417 | b.Stamps = nil |
| 418 | w.logf("box %d lost its local stamp record", b.ID) |
| 419 | w.stats["stamp records lost"]++ |
| 420 | } |
| 421 | if !on("floor-boot-recovery") && w.rng.Intn(3) == 0 { |
| 422 | b.Old = true |
| 423 | w.logf("box %d boot-recovered an old executable", b.ID) |
| 424 | } |
| 425 | w.logf("box %d rebooted", b.ID) |
| 426 | case 2: |
| 427 | if w.fail["partition"] { |
| 428 | b.Partitioned = !b.Partitioned |
| 429 | } |
| 430 | case 3: |
| 431 | if w.fail["freezetgt"] { |
| 432 | // the biased schedule (rev 7.5): freeze the landing target in |
| 433 | // its probe or settle window, and keep it frozen until the |
| 434 | // source runs again, if a declare-lost let it |
| 435 | c := w.claim |
| 436 | if c != nil && c.Op.Kind == "landing" && b.ID == c.Op.Tgt && !b.Frozen && (c.Op.LandPhase == "probing" || c.Op.LandPhase == "probed") { |
| 437 | b.Frozen = true |
| 438 | w.logf("box %d frozen in its settle window (biased schedule)", b.ID) |
| 439 | break |
| 440 | } |
| 441 | if b.Frozen && c != nil && (c.Op.Kind == "landing" || c.Op.Kind == "restoring") && b.ID == c.Op.Tgt { |
| 442 | if s := w.boxByID(c.Op.Src); s != nil && hasMain(s) { |
| 443 | b.Frozen = false |
| 444 | w.logf("box %d resumed beside the recovered source (biased schedule)", b.ID) |
| 445 | } |
| 446 | break |
| 447 | } |
| 448 | } |
| 449 | if w.fail["freeze"] { |
| 450 | b.Frozen = !b.Frozen |
| 451 | } |
| 452 | case 4: |
| 453 | if w.fail["pause"] { |
| 454 | b.AgentPaused = !b.AgentPaused |
| 455 | } |
| 456 | case 5: |
| 457 | c := w.claim |
| 458 | committed := c != nil && (c.Op.Committed || w.rec != nil && w.rec.MoveID == c.Op.ID && w.rec.MoveState == "committed") |
| 459 | if w.fail["destroy"] && c != nil && (c.Op.Kind == "landing" || c.Op.Kind == "restoring") && !committed && b.ID == c.Op.Tgt { |
| 460 | b.Destroyed = true |
| 461 | b.Procs = nil |
| 462 | w.logf("box %d destroyed", b.ID) |
| 463 | } |
| 464 | case 6: |
| 465 | w.console = nil // console crash: plans dropped |
| 466 | case 7: |
| 467 | // the serving box loses its database file while the app is idle: the |
| 468 | // next wake restores cold. Writes after the last completed |
| 469 | // checkpoint are lost by design (the RPO), and declared so; every |
| 470 | // durable write must come back (§14.2.3). |
| 471 | c := w.claim |
| 472 | if w.fail["disk"] && c != nil && c.Holder != nil && c.Op.Kind == "serving" && c.Op.Exec == b.ID && b.DB != nil && !hasMain(b) && len(b.Procs) == 0 { |
| 473 | // physical loss only (R10-6): the next admission declares |
| 474 | b.DB = nil |
| 475 | w.stats["disk-lost"]++ |
| 476 | w.logf("box %d lost its database file", b.ID) |
| 477 | } |
| 478 | default: |
| 479 | w.completeDue(false) |
| 480 | } |
| 481 | } |
| 482 | |
| 483 | func (w *world) replaceDestroyed() { |
| 484 | for r := 0; r < nRegions; r++ { |
| 485 | for len(w.regionBoxes(r)) < w.regionMin(r) { // the region's minimum (H5): the group launches replacements |
| 486 | w.nextID++ |
| 487 | nb := &box{ID: 100 + w.nextID, Region: r, HasKey: true, Enrolled: w.perRegion == 1, EnrollAt: w.now, DeleteOnTermination: true} |
| 488 | nb.forget() |
| 489 | w.boxes = append(w.boxes, nb) |
| 490 | w.logf("box %d replaces the destroyed box in region %d", nb.ID, r) |
| 491 | } |
| 492 | } |
| 493 | } |
| 494 | |
amadan