| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "io/fs" |
| 9 | "os" |
| 10 | "path/filepath" |
| 11 | "strings" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/lease" |
| 16 | "github.com/carlosframework/platform/internal/moves" |
| 17 | "github.com/carlosframework/platform/internal/provision" |
| 18 | "github.com/carlosframework/platform/internal/release" |
| 19 | ) |
| 20 | |
| 21 | // pagesWith counts the pages whose subject starts with prefix. |
| 22 | func pagesWith(pages pagesSeen, prefix string) (n int, level string) { |
| 23 | for _, p := range pages { |
| 24 | if strings.HasPrefix(p.subject, prefix) { |
| 25 | n++ |
| 26 | level = p.level |
| 27 | } |
| 28 | } |
| 29 | return n, level |
| 30 | } |
| 31 | |
| 32 | // capturePagingLog records the paging code's agent log lines for the test. |
| 33 | func capturePagingLog(t *testing.T) *[]string { |
| 34 | t.Helper() |
| 35 | var lines []string |
| 36 | saved := pagingLogf |
| 37 | pagingLogf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) } |
| 38 | t.Cleanup(func() { pagingLogf = saved }) |
| 39 | return &lines |
| 40 | } |
| 41 | |
| 42 | // pageDelivered reports whether the page outbox under dataDir holds rule's |
| 43 | // entry for (host, op), delivered. |
| 44 | func pageDelivered(t *testing.T, dataDir, host, op, rule string) bool { |
| 45 | t.Helper() |
| 46 | e, err := readPageEntry(filepath.Join(pageEntryDir(dataDir, host, op), rule+".json")) |
| 47 | if errors.Is(err, os.ErrNotExist) { |
| 48 | return false |
| 49 | } |
| 50 | if err != nil { |
| 51 | t.Fatal(err) |
| 52 | } |
| 53 | return e.Delivered != nil |
| 54 | } |
| 55 | |
| 56 | // memoryPending counts the pages the index holds for dataDir that the |
| 57 | // outbox could not keep and that are not delivered yet. |
| 58 | func memoryPending(dataDir string) int { |
| 59 | pageOutboxMu.Lock() |
| 60 | defer pageOutboxMu.Unlock() |
| 61 | n := 0 |
| 62 | for key, item := range pageIndex { |
| 63 | if key.dataDir == dataDir && item.memory && !item.delivered { |
| 64 | n++ |
| 65 | } |
| 66 | } |
| 67 | return n |
| 68 | } |
| 69 | |
| 70 | // tFlush is one pass's page flush, with no record listing. |
| 71 | func tFlush(ctx context.Context, w *movesWorker) { w.flushPageOutbox(ctx) } |
| 72 | |
| 73 | // readOnlyDir makes dir and everything under it unwritable until the test |
| 74 | // ends (or restore is called). |
| 75 | func readOnlyDir(t *testing.T, dir string) (restore func()) { |
| 76 | t.Helper() |
| 77 | var dirs []string |
| 78 | filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { |
| 79 | if err == nil && d.IsDir() { |
| 80 | dirs = append(dirs, p) |
| 81 | } |
| 82 | return nil |
| 83 | }) |
| 84 | for _, d := range dirs { |
| 85 | if err := os.Chmod(d, 0o500); err != nil { |
| 86 | t.Fatal(err) |
| 87 | } |
| 88 | } |
| 89 | restore = func() { |
| 90 | for _, d := range dirs { |
| 91 | os.Chmod(d, 0o700) |
| 92 | } |
| 93 | } |
| 94 | t.Cleanup(restore) |
| 95 | return restore |
| 96 | } |
| 97 | |
| 98 | // unreachableStore answers every read and listing with an error, as a bucket |
| 99 | // that cannot be reached. |
| 100 | type unreachableStore struct{ release.Store } |
| 101 | |
| 102 | var errBucketDown = errors.New("the bucket cannot be reached") |
| 103 | |
| 104 | func (unreachableStore) Get(context.Context, string) ([]byte, string, error) { |
| 105 | return nil, "", errBucketDown |
| 106 | } |
| 107 | func (unreachableStore) List(context.Context, string) ([]string, error) { return nil, errBucketDown } |
| 108 | func (unreachableStore) ListDirs(context.Context, string) ([]string, error) { |
| 109 | return nil, errBucketDown |
| 110 | } |
| 111 | |
| 112 | // refusingPager is a pager that refuses while *refuse is set, recording |
| 113 | // what it takes into pages. |
| 114 | func refusingPager(pages *pagesSeen, refuse *bool) pageFunc { |
| 115 | return func(ctx context.Context, level, subject, body string) error { |
| 116 | if *refuse { |
| 117 | return errors.New("the webhook is down") |
| 118 | } |
| 119 | return pages.hook()(ctx, level, subject, body) |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | // TestPagingIsRateLimited (Review Focus 4; spec §13; preflight L8): a rule |
| 124 | // that fires every pass must not flood. On the box, stuck pages once per |
| 125 | // entry into stuck (a retry that sticks again pages again), rolled_back once |
| 126 | // per op, a stray process at the stop once per op, and a refused start check |
| 127 | // once per op; each rule's state is the op journal, so a restarted agent |
| 128 | // does not page it again. The console's own rules are tested where they |
| 129 | // live: rolling_back past release_deadline once an hour |
| 130 | // (TestConsolePagesThroughAlerting), preparing once |
| 131 | // (TestRecoveryPassAbortsAfterSixHours) from moves.PreparingSince |
| 132 | // (TestPreparingPageRunsFromPreparingSince). |
| 133 | func TestPagingIsRateLimited(t *testing.T) { |
| 134 | ctx := context.Background() |
| 135 | |
| 136 | t.Run("stuck, once per entry", func(t *testing.T) { |
| 137 | dd := t.TempDir() |
| 138 | var pages pagesSeen |
| 139 | worker := func() *movesWorker { |
| 140 | return &movesWorker{cfg: moveCfg{DataDir: dd, Box: "uk-box", Page: pages.hook(), Now: func() time.Time { return time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) }}} |
| 141 | } |
| 142 | rec := provision.Record{Host: p1Host, Move: &provision.MoveState{ID: p1ID, State: provision.MoveRollingBack, |
| 143 | Rollback: &provision.MoveRollback{ID: "rb_abcdefghijklmnopqrstuvwxyz", TargetPhase: provision.RbTargetReturning}}} |
| 144 | w := worker() |
| 145 | for i := 0; i < 3; i++ { |
| 146 | w.pageStuck(ctx, rec, "target", provision.StuckReturnTargetDatabase) |
| 147 | } |
| 148 | worker().pageStuck(ctx, rec, "target", provision.StuckReturnTargetDatabase) // a restarted agent |
| 149 | if n, level := pagesWith(pages, "carlos move stuck: "); n != 1 || level != "incident" { |
| 150 | t.Fatalf("stuck pages %d (%s), want one incident: %+v", n, level, pages) |
| 151 | } |
| 152 | if !pageDelivered(t, dd, p1Host, p1ID, "stuck-0") { |
| 153 | t.Fatal("the outbox does not hold the stuck page delivered") |
| 154 | } |
| 155 | // An operator's retry advances the epoch; stuck again is a new entry. |
| 156 | rec.Move.RecoveryRetryEpoch = 1 |
| 157 | w.pageStuck(ctx, rec, "target", provision.StuckReturnTargetDatabase) |
| 158 | w.pageStuck(ctx, rec, "target", provision.StuckReturnTargetDatabase) |
| 159 | if n, _ := pagesWith(pages, "carlos move stuck: "); n != 2 { |
| 160 | t.Fatalf("stuck pages after a retry %d, want two: %+v", n, pages) |
| 161 | } |
| 162 | // One entry into stuck is one page, whichever half fires it. |
| 163 | w.pageStuck(ctx, rec, "source", provision.StuckBaselineMismatch) |
| 164 | if n, _ := pagesWith(pages, "carlos move stuck: "); n != 2 || !pageDelivered(t, dd, p1Host, p1ID, "stuck-1") { |
| 165 | t.Fatalf("stuck pages %d, want still two, the second delivered: %+v", n, pages) |
| 166 | } |
| 167 | }) |
| 168 | |
| 169 | t.Run("rolled_back, once per op, at the recovery's close", func(t *testing.T) { |
| 170 | h := newRecoverHarness(t, false) |
| 171 | h.released(t) |
| 172 | h.srcPass(t) |
| 173 | h.opened(t) |
| 174 | h.closeOut(t) |
| 175 | if rec := h.read(t, p1Host); rec.Move != nil || rec.LastMove == nil || rec.LastMove.Outcome != "rolled_back" { |
| 176 | t.Fatalf("not closed: %+v %+v", rec.Move, rec.LastMove) |
| 177 | } |
| 178 | h.srcPass(t) |
| 179 | h.sw = nil // a restarted agent |
| 180 | h.srcPass(t) |
| 181 | h.rollbackPass(t) |
| 182 | n, level := pagesWith(h.pages, "carlos move rolled back: ") |
| 183 | if n != 1 || level != "warning" { |
| 184 | t.Fatalf("rolled_back pages %d (%s), want one warning: %+v", n, level, h.pages) |
| 185 | } |
| 186 | want := "The move " + p1ID + " of " + p1Host + " rolled back, and " + p1Host + " is serving on ie-box again." |
| 187 | for _, p := range h.pages { |
| 188 | if strings.HasPrefix(p.subject, "carlos move rolled back: ") && p.body != want { |
| 189 | t.Fatalf("the page's body %q, want %q", p.body, want) |
| 190 | } |
| 191 | } |
| 192 | }) |
| 193 | |
| 194 | t.Run("rolled_back, once per op, at the revert's close", func(t *testing.T) { |
| 195 | h := newRevertHarness(t) |
| 196 | h.fromStopped(t) |
| 197 | h.arm(t) |
| 198 | var pages pagesSeen |
| 199 | h.cfg.Page = pages.hook() |
| 200 | h.drive(t) |
| 201 | h.closed(t) |
| 202 | h.pass(t) |
| 203 | if n, level := pagesWith(pages, "carlos move rolled back: "); n != 1 || level != "warning" { |
| 204 | t.Fatalf("rolled_back pages %d (%s), want one warning: %+v", n, level, pages) |
| 205 | } |
| 206 | }) |
| 207 | |
| 208 | t.Run("a stray process at the stop, once per op, with M-C20 to operators only", func(t *testing.T) { |
| 209 | logged := capturePagingLog(t) |
| 210 | h := newCutHarness(t) |
| 211 | h.ready(t) |
| 212 | var pages pagesSeen |
| 213 | sink := &memberSink{} |
| 214 | h.cfg.Page, h.cfg.Events = pages.hook(), sink |
| 215 | evidence := h.cfg.Evidence |
| 216 | h.cfg.Evidence = func(ctx context.Context, host, location string, dbs []string) (moves.StoppedReport, error) { |
| 217 | r, err := evidence(ctx, host, location, dbs) |
| 218 | r.Steps[moves.StepStray] = moves.Check{Reason: "1 process(es) besides the agent in its leaf"} |
| 219 | r.Stray = []int{4242} |
| 220 | return r, err |
| 221 | } |
| 222 | h.cut(t) |
| 223 | for i := 0; i < 3; i++ { |
| 224 | h.pass(t) |
| 225 | } |
| 226 | h.freshWorker() // a restarted agent |
| 227 | h.pass(t) |
| 228 | if m := h.move(t); m.CutPhase != provision.CutStopping || h.ev.calls < 4 { |
| 229 | t.Fatalf("the stray process did not hold the cut at stopping: %+v after %d evidence calls", m, h.ev.calls) |
| 230 | } |
| 231 | if n, level := pagesWith(pages, "carlos move stray process: "); n != 1 || level != "incident" { |
| 232 | t.Fatalf("stray pages %d (%s), want one incident: %+v", n, level, pages) |
| 233 | } |
| 234 | // Fix round 1, ruling 1: M-C20 is operator copy. It is the page's |
| 235 | // body and the agent's log line, and the app's members are told |
| 236 | // nothing. |
| 237 | mc20 := "ie-box is running a process for " + p1Host + " that this move cannot account for. An operator has been paged." |
| 238 | for _, p := range pages { |
| 239 | if strings.HasPrefix(p.subject, "carlos move stray process: ") && p.body != mc20 { |
| 240 | t.Fatalf("the stray page's body %q, want M-C20 %q", p.body, mc20) |
| 241 | } |
| 242 | } |
| 243 | if len(sink.events) != 0 { |
| 244 | t.Fatalf("the app's members were told %q; M-C20 is the operators'", sink.events) |
| 245 | } |
| 246 | if n := strings.Count(strings.Join(*logged, "\n"), mc20); n != 1 { |
| 247 | t.Fatalf("the agent's log says M-C20 %d times, want once: %q", n, *logged) |
| 248 | } |
| 249 | if !pageDelivered(t, h.cfg.DataDir, p1Host, p1ID, pageRuleStray) { |
| 250 | t.Fatal("the cut's journal does not record the stray page") |
| 251 | } |
| 252 | // Evidence that fails for any other reason pages nothing. |
| 253 | h2 := newCutHarness(t) |
| 254 | h2.ready(t) |
| 255 | var pages2 pagesSeen |
| 256 | h2.cfg.Page = pages2.hook() |
| 257 | h2.ev.stopped = false // the database is still held open: StepFDs |
| 258 | h2.cut(t) |
| 259 | h2.pass(t) |
| 260 | h2.pass(t) |
| 261 | if len(pages2) != 0 { |
| 262 | t.Fatalf("an unproven stop with no stray process paged: %+v", pages2) |
| 263 | } |
| 264 | }) |
| 265 | |
| 266 | t.Run("a refused start check, once per op", func(t *testing.T) { |
| 267 | f := newPOFixture(t) |
| 268 | var pages pagesSeen |
| 269 | f.owners.Page = pages.hook() |
| 270 | o := f.read() |
| 271 | in, err := f.owners.Select(ctx, f.route, o, poProbe) |
| 272 | if err != nil { |
| 273 | t.Fatal(err) |
| 274 | } |
| 275 | admitted := f.admit(o, in) |
| 276 | spawn := poWork{action: "spawn", f: f} |
| 277 | f.writeLocal("mv_other", poOp, f.inc) |
| 278 | for i := 0; i < 3; i++ { |
| 279 | if err := f.owners.Materialize(ctx, admitted, in, spawn); !errors.Is(err, moves.ErrStartFiles) { |
| 280 | t.Fatalf("files of another op: %v", err) |
| 281 | } |
| 282 | } |
| 283 | // A restarted agent: the owners' memory is gone, the journal is not. |
| 284 | f.owners.mu.Lock() |
| 285 | f.owners.published, f.owners.fences = nil, nil |
| 286 | f.owners.mu.Unlock() |
| 287 | f.writeLocal(poOp, "src", f.inc) |
| 288 | if err := f.owners.Materialize(ctx, admitted, in, spawn); err == nil { |
| 289 | t.Fatal("a database of another generation started") |
| 290 | } |
| 291 | if n, level := pagesWith(pages, "carlos move start refused: "); n != 1 || level != "incident" { |
| 292 | t.Fatalf("start check pages %d (%s), want one incident: %+v", n, level, pages) |
| 293 | } |
| 294 | if !pageDelivered(t, f.dataDir, poHost, poOp, pageRuleStartCheck) { |
| 295 | t.Fatal("the landing's journal does not record the start check page") |
| 296 | } |
| 297 | }) |
| 298 | |
| 299 | // Fix round 1, ruling 2: production stays unwired until Plan 5 9b, and |
| 300 | // an unwired pager is never silent. Each rule says so in the agent's |
| 301 | // log, loudly, once per move while the agent runs (fix round 2: the |
| 302 | // worker's memory, not the journal); nothing is refused or kept. The probe owners' refused start check says what |
| 303 | // became of its page, so the result is never dropped. |
| 304 | t.Run("an unwired pager is loud, once per rule per move", func(t *testing.T) { |
| 305 | logged := capturePagingLog(t) |
| 306 | dd := t.TempDir() |
| 307 | worker := func() *movesWorker { return &movesWorker{cfg: moveCfg{DataDir: dd, Box: "uk-box"}} } |
| 308 | rec := provision.Record{Host: p1Host, Move: &provision.MoveState{ID: p1ID, State: provision.MoveStuck}} |
| 309 | loud := func() int { return strings.Count(strings.Join(*logged, "\n"), "WARNING: no operator pager is wired") } |
| 310 | w := worker() |
| 311 | w.pageStuck(ctx, rec, "target", provision.StuckReturnTargetDatabase) |
| 312 | w.pageStuck(ctx, rec, "target", provision.StuckReturnTargetDatabase) |
| 313 | w.pageStuck(ctx, rec, "target", provision.StuckReturnTargetDatabase) |
| 314 | if loud() != 1 { |
| 315 | t.Fatalf("the unwired stuck page warned %d times, want once: %q", loud(), *logged) |
| 316 | } |
| 317 | w.pageRolledBack(ctx, rec) |
| 318 | w.pageRolledBack(ctx, rec) |
| 319 | if loud() != 2 { |
| 320 | t.Fatalf("a second rule warned %d times in all, want 2: %q", loud(), *logged) |
| 321 | } |
| 322 | for _, l := range *logged { |
| 323 | if strings.Contains(l, "WARNING") && (!strings.Contains(l, p1Host) || !strings.Contains(l, p1ID)) { |
| 324 | t.Fatalf("the warning does not name the host and the move: %q", l) |
| 325 | } |
| 326 | } |
| 327 | |
| 328 | // The probe owners: the start check is refused as before, the |
| 329 | // warning comes once, and every refusal's line says it was not paged. |
| 330 | *logged = nil |
| 331 | f := newPOFixture(t) |
| 332 | o := f.read() |
| 333 | in, err := f.owners.Select(ctx, f.route, o, poProbe) |
| 334 | if err != nil { |
| 335 | t.Fatal(err) |
| 336 | } |
| 337 | admitted := f.admit(o, in) |
| 338 | f.writeLocal("mv_other", poOp, f.inc) |
| 339 | for i := 0; i < 2; i++ { |
| 340 | if err := f.owners.Materialize(ctx, admitted, in, poWork{action: "spawn", f: f}); !errors.Is(err, moves.ErrStartFiles) { |
| 341 | t.Fatalf("files of another op: %v", err) |
| 342 | } |
| 343 | } |
| 344 | if loud() != 1 { |
| 345 | t.Fatalf("the unwired start check page warned %d times, want once: %q", loud(), *logged) |
| 346 | } |
| 347 | if n := strings.Count(strings.Join(*logged, "\n"), "refused to start "+poHost+" for "+poOp+" (not paged: no pager is wired)"); n != 2 { |
| 348 | t.Fatalf("the refusals' lines %d, want 2 saying the page was not sent: %q", n, *logged) |
| 349 | } |
| 350 | if _, err := os.Stat(pageOutboxDir(dd)); !os.IsNotExist(err) { |
| 351 | t.Fatalf("an unwired worker kept pages: %v", err) |
| 352 | } |
| 353 | }) |
| 354 | |
| 355 | // Fix round 2, ruling 1: the page outbox. A refused page is sent again |
| 356 | // by a later pass, whatever its move has done since; it is never lost |
| 357 | // to the journal being parked; a double fault keeps it in memory; a |
| 358 | // rule that fires again after delivery never pages again. |
| 359 | t.Run("outbox: a refused stuck page is sent by a later pass", func(t *testing.T) { |
| 360 | h := newRevertHarness(t) |
| 361 | h.fromStopped(t) |
| 362 | h.arm(t) |
| 363 | var pages pagesSeen |
| 364 | refuse := true |
| 365 | h.cfg.Page = refusingPager(&pages, &refuse) |
| 366 | if err := os.WriteFile(filepath.Join(h.cfg.DataDir, p1Host, "upload.txt"), []byte("changed while stopped"), 0o644); err != nil { |
| 367 | t.Fatal(err) |
| 368 | } |
| 369 | h.pass(t) |
| 370 | if m := h.move(t); m.State != provision.MoveStuck || m.StuckReason != provision.StuckBaselineMismatch { |
| 371 | t.Fatalf("not stuck: %+v", m) |
| 372 | } |
| 373 | if len(pages) != 0 || pageDelivered(t, h.cfg.DataDir, p1Host, p1ID, "stuck-0") { |
| 374 | t.Fatalf("a refused page counted as sent: %+v", pages) |
| 375 | } |
| 376 | refuse = false |
| 377 | h.pass(t) // inside the backoff: not yet |
| 378 | if len(pages) != 0 { |
| 379 | t.Fatalf("retried inside its backoff: %+v", pages) |
| 380 | } |
| 381 | h.clock.Advance(2 * time.Minute) |
| 382 | h.pass(t) // no new stuck event: the pass's flush sends it |
| 383 | pagedOnce(t, pages, "ie-box", "source", provision.StuckBaselineMismatch) |
| 384 | if !pageDelivered(t, h.cfg.DataDir, p1Host, p1ID, "stuck-0") { |
| 385 | t.Fatal("the flushed page is not marked delivered") |
| 386 | } |
| 387 | h.clock.Advance(time.Hour) |
| 388 | h.pass(t) |
| 389 | h.freshWorker() // a restarted agent |
| 390 | h.pass(t) |
| 391 | pagedOnce(t, pages, "ie-box", "source", provision.StuckBaselineMismatch) |
| 392 | }) |
| 393 | |
| 394 | t.Run("outbox: a refused rolled_back page is delivered after the journal is parked", func(t *testing.T) { |
| 395 | h := newRecoverHarness(t, false) |
| 396 | refuse := true |
| 397 | var pages pagesSeen |
| 398 | h.srcPage = refusingPager(&pages, &refuse) // the source's pager |
| 399 | h.released(t) |
| 400 | h.srcPass(t) |
| 401 | h.opened(t) |
| 402 | h.closeOut(t) |
| 403 | if rec := h.read(t, p1Host); rec.Move != nil || rec.LastMove == nil || rec.LastMove.Outcome != "rolled_back" { |
| 404 | t.Fatalf("not closed: %+v %+v", rec.Move, rec.LastMove) |
| 405 | } |
| 406 | if live, parked := h.parkedJournal(t, p1ID); live || !parked { |
| 407 | t.Fatalf("the source's journal of the move: live %v, parked %v; want parked", live, parked) |
| 408 | } |
| 409 | if n, _ := pagesWith(pages, "carlos move rolled back: "); n != 0 { |
| 410 | t.Fatalf("a refused page was delivered: %+v", pages) |
| 411 | } |
| 412 | refuse = false |
| 413 | h.clock.Advance(2 * time.Minute) |
| 414 | h.srcPass(t) |
| 415 | if n, level := pagesWith(pages, "carlos move rolled back: "); n != 1 || level != "warning" { |
| 416 | t.Fatalf("rolled_back pages after the park %d (%s), want one warning: %+v", n, level, pages) |
| 417 | } |
| 418 | // Delivered, and the move retired on this box (no record carries it, |
| 419 | // its journals are parked): the tombstone goes, at the next pass. |
| 420 | h.srcPass(t) |
| 421 | if _, err := os.Stat(pageEntryDir(h.sdir, p1Host, p1ID)); !os.IsNotExist(err) { |
| 422 | t.Fatalf("the retired move's tombstone is still kept: %v", err) |
| 423 | } |
| 424 | h.clock.Advance(time.Hour) |
| 425 | h.srcPass(t) |
| 426 | if n, _ := pagesWith(pages, "carlos move rolled back: "); n != 1 { |
| 427 | t.Fatalf("rolled_back paged again: %+v", pages) |
| 428 | } |
| 429 | }) |
| 430 | |
| 431 | t.Run("outbox: the outbox write and the delivery fail, a later pass delivers from memory", func(t *testing.T) { |
| 432 | logged := capturePagingLog(t) |
| 433 | dd := t.TempDir() |
| 434 | // The outbox cannot be made: a file stands where its directory goes. |
| 435 | if err := os.MkdirAll(filepath.Join(dd, ".moves"), 0o700); err != nil { |
| 436 | t.Fatal(err) |
| 437 | } |
| 438 | if err := os.WriteFile(pageOutboxDir(dd), []byte("in the way"), 0o600); err != nil { |
| 439 | t.Fatal(err) |
| 440 | } |
| 441 | now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) |
| 442 | var pages pagesSeen |
| 443 | refuse := true |
| 444 | w := &movesWorker{cfg: moveCfg{DataDir: dd, Box: "uk-box", Now: func() time.Time { return now }, Page: refusingPager(&pages, &refuse)}} |
| 445 | if o := w.firePage(ctx, p1Host, p1ID, "stuck-0", "incident", "carlos move stuck: "+p1Host, "b"); o != pageUndelivered { |
| 446 | t.Fatalf("the double fault: %s", o) |
| 447 | } |
| 448 | if memoryPending(dd) != 1 || !strings.Contains(strings.Join(*logged, "\n"), "kept in memory only") { |
| 449 | t.Fatalf("the page is not kept in memory, loudly: %d %q", memoryPending(dd), *logged) |
| 450 | } |
| 451 | refuse = false |
| 452 | tFlush(ctx, w) // inside the backoff |
| 453 | if len(pages) != 0 { |
| 454 | t.Fatalf("retried inside its backoff: %+v", pages) |
| 455 | } |
| 456 | now = now.Add(2 * time.Minute) |
| 457 | tFlush(ctx, w) |
| 458 | w.retirePageTombstones(nil) |
| 459 | if n, _ := pagesWith(pages, "carlos move stuck: "); n != 1 || memoryPending(dd) != 0 { |
| 460 | t.Fatalf("the memory's page: %d delivered, %d kept; want 1 and 0", n, memoryPending(dd)) |
| 461 | } |
| 462 | now = now.Add(time.Hour) |
| 463 | tFlush(ctx, w) |
| 464 | w.retirePageTombstones(nil) |
| 465 | if len(pages) != 1 { |
| 466 | t.Fatalf("a delivered memory page was sent again: %+v", pages) |
| 467 | } |
| 468 | }) |
| 469 | |
| 470 | t.Run("outbox: a rule that fires again after delivery does not page again", func(t *testing.T) { |
| 471 | dd := t.TempDir() |
| 472 | now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) |
| 473 | var pages pagesSeen |
| 474 | worker := func() *movesWorker { |
| 475 | return &movesWorker{cfg: moveCfg{DataDir: dd, Box: "ie-box", Now: func() time.Time { return now }, Page: pages.hook()}} |
| 476 | } |
| 477 | rec := provision.Record{Host: p1Host, Move: &provision.MoveState{ID: p1ID, State: provision.MoveRollingBack}} |
| 478 | w := worker() |
| 479 | w.pageRolledBack(ctx, rec) |
| 480 | if !pageDelivered(t, dd, p1Host, p1ID, pageRuleRolledBack) { |
| 481 | t.Fatal("not delivered") |
| 482 | } |
| 483 | // The record still carries the move: the tombstone stays through |
| 484 | // passes, and a firing again, here or after a restart, finds it. |
| 485 | live := []provision.Record{rec} |
| 486 | for i := 0; i < 3; i++ { |
| 487 | now = now.Add(time.Hour) |
| 488 | tFlush(ctx, w) |
| 489 | w.retirePageTombstones(live) |
| 490 | w.pageRolledBack(ctx, rec) |
| 491 | worker().pageRolledBack(ctx, rec) |
| 492 | } |
| 493 | if n, _ := pagesWith(pages, "carlos move rolled back: "); n != 1 { |
| 494 | t.Fatalf("rolled_back pages %d, want one: %+v", n, pages) |
| 495 | } |
| 496 | // A journal of the move on this box keeps it too. |
| 497 | j, err := moves.ReadJournal(dd, p1Host, p1ID) |
| 498 | if err != nil { |
| 499 | t.Fatal(err) |
| 500 | } |
| 501 | j.Mark("x", "y") |
| 502 | if err := j.Commit(dd); err != nil { |
| 503 | t.Fatal(err) |
| 504 | } |
| 505 | tFlush(ctx, w) |
| 506 | w.retirePageTombstones(nil) |
| 507 | if !pageDelivered(t, dd, p1Host, p1ID, pageRuleRolledBack) { |
| 508 | t.Fatal("the tombstone went while the move's journal is on the box") |
| 509 | } |
| 510 | }) |
| 511 | |
| 512 | t.Run("outbox: a flush-delivered M-C20 still has its box log line", func(t *testing.T) { |
| 513 | logged := capturePagingLog(t) |
| 514 | h := newCutHarness(t) |
| 515 | h.ready(t) |
| 516 | var pages pagesSeen |
| 517 | refuse := true |
| 518 | h.cfg.Page = refusingPager(&pages, &refuse) |
| 519 | evidence := h.cfg.Evidence |
| 520 | h.cfg.Evidence = func(ctx context.Context, host, location string, dbs []string) (moves.StoppedReport, error) { |
| 521 | r, err := evidence(ctx, host, location, dbs) |
| 522 | r.Steps[moves.StepStray] = moves.Check{Reason: "1 process(es) besides the agent in its leaf"} |
| 523 | r.Stray = []int{4242} |
| 524 | return r, err |
| 525 | } |
| 526 | h.cut(t) |
| 527 | h.pass(t) |
| 528 | mc20 := "ie-box is running a process for " + p1Host + " that this move cannot account for. An operator has been paged." |
| 529 | if n := strings.Count(strings.Join(*logged, "\n"), mc20); n != 1 { |
| 530 | t.Fatalf("M-C20 in the agent's log %d times at the detection, want once: %q", n, *logged) |
| 531 | } |
| 532 | refuse = false |
| 533 | h.clock.Advance(2 * time.Minute) |
| 534 | h.pass(t) |
| 535 | h.pass(t) |
| 536 | if n, _ := pagesWith(pages, "carlos move stray process: "); n != 1 || !pageDelivered(t, h.cfg.DataDir, p1Host, p1ID, pageRuleStray) { |
| 537 | t.Fatalf("the stray page after the flush: %+v", pages) |
| 538 | } |
| 539 | if n := strings.Count(strings.Join(*logged, "\n"), mc20); n != 1 { |
| 540 | t.Fatalf("M-C20 in the agent's log %d times, want once: %q", n, *logged) |
| 541 | } |
| 542 | }) |
| 543 | |
| 544 | // Fix round 3: the outbox's failure paths. An in-memory index over the |
| 545 | // disk outbox keeps each (host, move, rule) once, through any storage |
| 546 | // failure. |
| 547 | t.Run("index: a delivered tombstone on read-only storage stops every re-fire", func(t *testing.T) { |
| 548 | dd := t.TempDir() |
| 549 | now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) |
| 550 | var pages pagesSeen |
| 551 | w := &movesWorker{cfg: moveCfg{DataDir: dd, Box: "ie-box", Now: func() time.Time { return now }, Page: pages.hook()}} |
| 552 | rec := provision.Record{Host: p1Host, Move: &provision.MoveState{ID: p1ID, State: provision.MoveStuck}} |
| 553 | w.pageStuck(ctx, rec, "source", provision.StuckBaselineMismatch) |
| 554 | if !pageDelivered(t, dd, p1Host, p1ID, "stuck-0") { |
| 555 | t.Fatal("not delivered") |
| 556 | } |
| 557 | readOnlyDir(t, pageOutboxDir(dd)) |
| 558 | for i := 0; i < 3; i++ { |
| 559 | w.pageStuck(ctx, rec, "source", provision.StuckBaselineMismatch) |
| 560 | (&movesWorker{cfg: w.cfg}).pageStuck(ctx, rec, "source", provision.StuckBaselineMismatch) |
| 561 | } |
| 562 | if len(pages) != 1 { |
| 563 | t.Fatalf("re-fires on read-only storage paged again: %d pages", len(pages)) |
| 564 | } |
| 565 | }) |
| 566 | |
| 567 | t.Run("index: a backoff the disk cannot record still holds", func(t *testing.T) { |
| 568 | dd := t.TempDir() |
| 569 | now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) |
| 570 | attempts := 0 |
| 571 | refuse := func(context.Context, string, string, string) error { |
| 572 | attempts++ |
| 573 | return errors.New("the webhook is down") |
| 574 | } |
| 575 | w := &movesWorker{cfg: moveCfg{DataDir: dd, Box: "ie-box", Now: func() time.Time { return now }, Page: refuse}} |
| 576 | rec := provision.Record{Host: p1Host, Move: &provision.MoveState{ID: p1ID, State: provision.MoveStuck}} |
| 577 | w.pageStuck(ctx, rec, "source", provision.StuckBaselineMismatch) // attempt 1, next in 1 minute |
| 578 | readOnlyDir(t, pageOutboxDir(dd)) |
| 579 | now = now.Add(2 * time.Minute) |
| 580 | tFlush(ctx, w) // attempt 2: the disk cannot record its backoff (2 minutes) |
| 581 | if attempts != 2 { |
| 582 | t.Fatalf("attempts %d, want 2", attempts) |
| 583 | } |
| 584 | for i := 0; i < 3; i++ { |
| 585 | now = now.Add(30 * time.Second) |
| 586 | tFlush(ctx, w) |
| 587 | } |
| 588 | if attempts != 2 { |
| 589 | t.Fatalf("retried inside a backoff the disk could not record: %d attempts", attempts) |
| 590 | } |
| 591 | now = now.Add(time.Minute) // past the 2-minute backoff |
| 592 | tFlush(ctx, w) |
| 593 | if attempts != 3 { |
| 594 | t.Fatalf("attempts after the backoff %d, want 3", attempts) |
| 595 | } |
| 596 | // The count the disk could not record is kept too (re-review 3): the |
| 597 | // third attempt's backoff is 4 minutes, not the 2 a stale disk count |
| 598 | // of one would give again. |
| 599 | now = now.Add(2*time.Minute + 30*time.Second) |
| 600 | tFlush(ctx, w) |
| 601 | if attempts != 3 { |
| 602 | t.Fatalf("the third attempt's backoff restarted from the disk's stale count: %d attempts at 2m30s", attempts) |
| 603 | } |
| 604 | now = now.Add(2 * time.Minute) // 4m30s after the third attempt |
| 605 | tFlush(ctx, w) |
| 606 | if attempts != 4 { |
| 607 | t.Fatalf("attempts after the 4-minute backoff %d, want 4", attempts) |
| 608 | } |
| 609 | }) |
| 610 | |
| 611 | t.Run("index: stray fires in a double fault page once, and once the storage is back the tombstone is written", func(t *testing.T) { |
| 612 | dd := t.TempDir() |
| 613 | blocker := pageOutboxDir(dd) |
| 614 | if err := os.MkdirAll(filepath.Dir(blocker), 0o700); err != nil { |
| 615 | t.Fatal(err) |
| 616 | } |
| 617 | if err := os.WriteFile(blocker, []byte("in the way"), 0o600); err != nil { |
| 618 | t.Fatal(err) |
| 619 | } |
| 620 | now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) |
| 621 | var pages pagesSeen |
| 622 | refuse := true |
| 623 | w := &movesWorker{cfg: moveCfg{DataDir: dd, Box: "ie-box", Now: func() time.Time { return now }, Page: refusingPager(&pages, &refuse)}} |
| 624 | rec := provision.Record{Host: p1Host, Move: &provision.MoveState{ID: p1ID, State: provision.MoveCutting}} |
| 625 | for i := 0; i < 3; i++ { |
| 626 | w.pageStray(ctx, rec) |
| 627 | } |
| 628 | refuse = false |
| 629 | for i := 0; i < 3; i++ { |
| 630 | w.pageStray(ctx, rec) // re-fires while the page waits in memory |
| 631 | } |
| 632 | now = now.Add(2 * time.Minute) |
| 633 | tFlush(ctx, w) |
| 634 | tFlush(ctx, w) |
| 635 | w.pageStray(ctx, rec) |
| 636 | if n, _ := pagesWith(pages, "carlos move stray process: "); n != 1 { |
| 637 | t.Fatalf("stray pages during the double fault %d, want one: %+v", n, pages) |
| 638 | } |
| 639 | // The storage is back: the next pass writes the tombstone, and a |
| 640 | // re-fire, here or after a restart, does not page. |
| 641 | if err := os.Remove(blocker); err != nil { |
| 642 | t.Fatal(err) |
| 643 | } |
| 644 | tFlush(ctx, w) |
| 645 | if !pageDelivered(t, dd, p1Host, p1ID, pageRuleStray) { |
| 646 | t.Fatal("the delivered page's tombstone was not written once the storage came back") |
| 647 | } |
| 648 | w.pageStray(ctx, rec) |
| 649 | (&movesWorker{cfg: w.cfg}).pageStray(ctx, rec) |
| 650 | tFlush(ctx, w) |
| 651 | if n, _ := pagesWith(pages, "carlos move stray process: "); n != 1 { |
| 652 | t.Fatalf("stray pages after the storage came back %d, want still one: %+v", n, pages) |
| 653 | } |
| 654 | }) |
| 655 | |
| 656 | t.Run("index: a pending page is delivered while records and the fleet cannot be read", func(t *testing.T) { |
| 657 | dd := t.TempDir() |
| 658 | now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) |
| 659 | var pages pagesSeen |
| 660 | refuse := true |
| 661 | w := &movesWorker{cfg: moveCfg{DataDir: dd, Box: "ie-box", Enabled: true, Now: func() time.Time { return now }, Page: refusingPager(&pages, &refuse)}, |
| 662 | mirrors: map[string]lease.MirrorStore{}} |
| 663 | rec := provision.Record{Host: p1Host, Move: &provision.MoveState{ID: p1ID, State: provision.MoveStuck}} |
| 664 | w.pageStuck(ctx, rec, "source", provision.StuckBaselineMismatch) |
| 665 | refuse = false |
| 666 | now = now.Add(2 * time.Minute) |
| 667 | if err := movesPass(ctx, unreachableStore{}, w); err == nil { |
| 668 | t.Fatal("a pass over an unreachable bucket succeeded") |
| 669 | } |
| 670 | if n, _ := pagesWith(pages, "carlos move stuck: "); n != 1 || !pageDelivered(t, dd, p1Host, p1ID, "stuck-0") { |
| 671 | t.Fatalf("the pending page with the bucket down: %+v", pages) |
| 672 | } |
| 673 | }) |
| 674 | } |
| 675 | |
| 676 | // outboxTree lists every file under dataDir's .moves directory, with its |
| 677 | // bytes, so a test can tell whether a pass changed anything there. |
| 678 | func outboxTree(t *testing.T, dataDir string) map[string]string { |
| 679 | t.Helper() |
| 680 | out := map[string]string{} |
| 681 | root := filepath.Join(dataDir, ".moves") |
| 682 | err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { |
| 683 | if errors.Is(err, fs.ErrNotExist) && p == root { |
| 684 | return fs.SkipAll |
| 685 | } |
| 686 | if err != nil || d.IsDir() { |
| 687 | return err |
| 688 | } |
| 689 | b, err := os.ReadFile(p) |
| 690 | if err != nil { |
| 691 | return err |
| 692 | } |
| 693 | rel, _ := filepath.Rel(root, p) |
| 694 | out[filepath.ToSlash(rel)] = string(b) |
| 695 | return nil |
| 696 | }) |
| 697 | if err != nil { |
| 698 | t.Fatal(err) |
| 699 | } |
| 700 | return out |
| 701 | } |
| 702 | |
| 703 | // Final review I1: with the switch off, a pass leaves both outboxes alone. |
| 704 | // The pending page, the delivered page's tombstone and the kept log entry |
| 705 | // stay on disk untouched, and nothing is sent or written, until the switch |
| 706 | // is on again; then the same pass delivers them. |
| 707 | func TestSwitchOffPassLeavesTheOutboxes(t *testing.T) { |
| 708 | ctx := context.Background() |
| 709 | dd := t.TempDir() |
| 710 | now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) |
| 711 | var pages pagesSeen |
| 712 | refuse := true |
| 713 | var logged []moves.LogEvent |
| 714 | logFails := true |
| 715 | w := &movesWorker{cfg: moveCfg{DataDir: dd, Box: "ie-box", Enabled: true, Now: func() time.Time { return now }, Page: refusingPager(&pages, &refuse), |
| 716 | LogEvent: func(_ context.Context, _ release.Store, _ provision.Record, ev moves.LogEvent) error { |
| 717 | if logFails { |
| 718 | return errors.New("the bucket is down") |
| 719 | } |
| 720 | logged = append(logged, ev) |
| 721 | return nil |
| 722 | }}, mirrors: map[string]lease.MirrorStore{}} |
| 723 | rec := provision.Record{Account: "bab", App: "shop", Host: p1Host, Move: &provision.MoveState{ID: p1ID, State: provision.MoveStuck, From: "ie", To: "uk"}} |
| 724 | // A page the pager refused: pending on disk. |
| 725 | w.pageStuck(ctx, rec, "source", provision.StuckBaselineMismatch) |
| 726 | // A delivered page of a move no record carries any more: a tombstone a |
| 727 | // pass with the switch on would retire. |
| 728 | refuse = false |
| 729 | other := provision.Record{Host: p1Host, Move: &provision.MoveState{ID: "mv_zyxwvutsrqponmlkjihgfedcba"}} |
| 730 | w.pageRolledBack(ctx, other) |
| 731 | pages = nil |
| 732 | // A log entry whose write failed: kept in the log outbox. |
| 733 | w.observeMove(ctx, nil, provision.Record{Account: "bab", App: "shop", Host: p1Host, Region: "ie", |
| 734 | Move: &provision.MoveState{ID: p1ID, State: provision.MovePreparing, From: "ie", To: "uk"}}) |
| 735 | before := outboxTree(t, dd) |
| 736 | if len(before) != 3 { |
| 737 | t.Fatalf("the fixture's outboxes %v, want a pending page, a tombstone and a kept log entry", before) |
| 738 | } |
| 739 | logFails = false |
| 740 | now = now.Add(time.Hour) // every backoff is over |
| 741 | |
| 742 | s := release.NewDirStore(t.TempDir()) |
| 743 | w.cfg.Enabled = false |
| 744 | if err := movesPass(ctx, s, w); err != nil { |
| 745 | t.Fatal(err) |
| 746 | } |
| 747 | if len(pages) != 0 || len(logged) != 0 { |
| 748 | t.Fatalf("a pass with the switch off sent pages %+v and wrote log entries %+v", pages, logged) |
| 749 | } |
| 750 | if after := outboxTree(t, dd); fmt.Sprint(after) != fmt.Sprint(before) { |
| 751 | t.Fatalf("a pass with the switch off changed the outboxes:\nbefore %v\nafter %v", before, after) |
| 752 | } |
| 753 | |
| 754 | w.cfg.Enabled = true |
| 755 | if err := movesPass(ctx, s, w); err != nil { |
| 756 | t.Fatal(err) |
| 757 | } |
| 758 | if n, _ := pagesWith(pages, "carlos move stuck: "); n != 1 || len(logged) != 1 || logged[0].Identity != "requested" { |
| 759 | t.Fatalf("with the switch on again: pages %+v, log entries %+v", pages, logged) |
| 760 | } |
| 761 | } |
| 762 | |
| 763 | // Final review I2: the page outbox's replace and its delivered mark are |
| 764 | // durable before they count. A failed directory sync after the replace is |
| 765 | // an error, and a delivered entry already on disk is synced before the mark |
| 766 | // is acknowledged. |
| 767 | func TestPageOutboxIsDurableBeforeItAcknowledges(t *testing.T) { |
| 768 | spy := spyOutboxSync(t) |
| 769 | dd := t.TempDir() |
| 770 | now := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) |
| 771 | e := pageEntry{V: 1, Host: p1Host, Op: p1ID, Rule: "stuck-0", Level: "incident", Subject: "s", Body: "b", At: now, Tries: 1, Next: now.Add(time.Minute)} |
| 772 | if err := writePageEntryFixture(dd, e); err != nil { |
| 773 | t.Fatal(err) |
| 774 | } |
| 775 | dir := pageEntryDir(dd, p1Host, p1ID) |
| 776 | file := pageEntryPath(dd, p1Host, p1ID, "stuck-0") |
| 777 | |
| 778 | spy.fail[dir] = true |
| 779 | e.Tries = 2 |
| 780 | if err := writePageEntry(dd, e); err == nil { |
| 781 | t.Fatal("a replace whose directory sync failed was acknowledged") |
| 782 | } |
| 783 | |
| 784 | spy.reset() |
| 785 | if err := markPageDelivered(dd, e, now); err != nil { |
| 786 | t.Fatal(err) |
| 787 | } |
| 788 | spy.reset() |
| 789 | spy.fail[file] = true |
| 790 | if err := markPageDelivered(dd, e, now); err == nil { |
| 791 | t.Fatal("a delivered entry whose sync failed was acknowledged") |
| 792 | } |
| 793 | spy.reset() |
| 794 | if err := markPageDelivered(dd, e, now); err != nil { |
| 795 | t.Fatal(err) |
| 796 | } |
| 797 | if spy.synced[file] == 0 || spy.synced[dir] == 0 { |
| 798 | t.Fatalf("a delivered entry was acknowledged without syncing it and its directory: %v", spy.synced) |
| 799 | } |
| 800 | } |
| 801 | |
| 802 | // writePageEntryFixture writes a page entry with no sync spy in the way. |
| 803 | func writePageEntryFixture(dataDir string, e pageEntry) error { |
| 804 | b, err := json.Marshal(e) |
| 805 | if err != nil { |
| 806 | return err |
| 807 | } |
| 808 | dir := pageEntryDir(dataDir, e.Host, e.Op) |
| 809 | if err := os.MkdirAll(dir, 0o700); err != nil { |
| 810 | return err |
| 811 | } |
| 812 | return os.WriteFile(filepath.Join(dir, e.Rule+".json"), b, 0o600) |
| 813 | } |
| 814 | |
| 815 | // Final re-review 1, I1: with the switch off a pass touches no outbox, index |
| 816 | // or memo state. Here the move log's memory remembers two moves; the store |
| 817 | // then holds one host's record closed and another's replaced by a new move, |
| 818 | // which a pass with the switch on would forget. Off, the memory is exactly |
| 819 | // what it was. |
| 820 | func TestSwitchOffPassLeavesTheLogMemo(t *testing.T) { |
| 821 | ctx := context.Background() |
| 822 | var got []moves.LogEvent |
| 823 | w := eventWorker(&got) |
| 824 | flow := doneFlow() |
| 825 | w.observeMove(ctx, nil, flow[0]) // acme.go.tito.io, mvEvMoveID: requested |
| 826 | other := mvEvRec(provision.MovePreparing, func(m *provision.MoveState) { m.ID = "mv_otherotherotherotherother" }) |
| 827 | other.Host = "b.example.com" |
| 828 | w.observeMove(ctx, nil, other) |
| 829 | keys := []string{flow[0].Host + "/" + mvEvMoveID + "/requested", other.Host + "/mv_otherotherotherotherother/requested"} |
| 830 | before := w.logged.len() |
| 831 | if before != 2 || !w.logged.seen(keys[0]) || !w.logged.seen(keys[1]) { |
| 832 | t.Fatalf("the fixture's memory: %d", before) |
| 833 | } |
| 834 | s := release.NewDirStore(t.TempDir()) |
| 835 | if err := provision.CreateRecord(ctx, s, flow[len(flow)-1]); err != nil { // closed |
| 836 | t.Fatal(err) |
| 837 | } |
| 838 | replaced := mvEvRec(provision.MovePreparing, func(m *provision.MoveState) { m.ID = "mv_nextnextnextnextnextnextne" }) |
| 839 | replaced.Host = other.Host |
| 840 | if err := provision.CreateRecord(ctx, s, replaced); err != nil { |
| 841 | t.Fatal(err) |
| 842 | } |
| 843 | w.cfg.Enabled = false |
| 844 | if err := movesPass(ctx, s, w); err != nil { |
| 845 | t.Fatal(err) |
| 846 | } |
| 847 | if n := w.logged.len(); n != before || !w.logged.seen(keys[0]) || !w.logged.seen(keys[1]) { |
| 848 | t.Fatalf("a pass with the switch off changed the move log's memory: %d identities, want %d", n, before) |
| 849 | } |
| 850 | w.cfg.Enabled = true |
| 851 | if err := movesPass(ctx, s, w); err != nil { |
| 852 | t.Fatal(err) |
| 853 | } |
| 854 | if w.logged.seen(keys[0]) || w.logged.seen(keys[1]) { |
| 855 | t.Fatal("with the switch on the closed and replaced moves are still remembered") |
| 856 | } |
| 857 | } |
| 858 | |
amadan