| 1 | # Proves the S3 replicas actually restore: pulls every database named in |
| 2 | # /etc/litestream.yml back from S3 into a private tmp, opens it, and runs an |
| 3 | # integrity check. An un-restored backup is not a backup. Runs daily via |
| 4 | # carlos-restore-verify.timer; a failure leaves this unit in state 'failed', |
| 5 | # where fleet sweeps see it loudly. Ported from titogo's |
| 6 | # deploy/systemd/titogo-restore-verify.service. |
| 7 | [Unit] |
| 8 | Description=carlos restore-verify (restore every S3 replica, integrity-check it) |
| 9 | After=network-online.target carlos-litestream.service |
| 10 | Wants=network-online.target |
| 11 | |
| 12 | [Service] |
| 13 | Type=oneshot |
| 14 | ExecStart=/opt/carlos/carlos ops restore-verify -config /etc/litestream.yml |
| 15 | EnvironmentFile=/data/litestream.env |
| 16 | # Restore into a DISK-backed directory, not the PrivateTmp tmpfs. Both are |
| 17 | # charged to this cgroup — page cache is accounted just like tmpfs, so the |
| 18 | # reported peak barely moves (flagship: 485.6M on tmpfs, 494.4M on disk, for a |
| 19 | # 412M tenant). What changes is what the ceiling DOES when the restore does not |
| 20 | # fit, and that is the whole point: |
| 21 | # |
| 22 | # tmpfs pages are unevictable -> hitting MemoryMax KILLS the drill |
| 23 | # page cache is reclaimable -> hitting MemoryMax THROTTLES it |
| 24 | # |
| 25 | # Measured on the flagship 2026-08-16, same 412M database, MemoryMax=128M: |
| 26 | # on tmpfs, oom-killed after 8.3s; on disk, still running after 10 minutes |
| 27 | # under reclaim (stopped by hand). So the restored file no longer has to FIT |
| 28 | # in the cgroup's memory — a tenant outgrowing the ceiling degrades instead of |
| 29 | # failing. |
| 30 | # |
| 31 | # It does NOT make size irrelevant: a badly undersized ceiling makes the drill |
| 32 | # crawl. Keep MemoryMax comfortably above the largest tenant. systemd creates |
| 33 | # and cleans %C/carlos-verify and binds it writable despite |
| 34 | # ProtectSystem=strict. |
| 35 | CacheDirectory=carlos-verify |
| 36 | Environment=TMPDIR=%C/carlos-verify |
| 37 | User=carlos |
| 38 | Group=carlos |
| 39 | CapabilityBoundingSet= |
| 40 | NoNewPrivileges=yes |
| 41 | ProtectSystem=strict |
| 42 | ProtectHome=yes |
| 43 | PrivateTmp=yes |
| 44 | PrivateDevices=yes |
| 45 | RestrictAddressFamilies=AF_INET AF_INET6 |
| 46 | MemoryDenyWriteExecute=yes |
| 47 | # Sized to sit ABOVE the largest single database, not to bound the process: |
| 48 | # restored bytes are charged here either way (see the TMPDIR note). Measured |
| 49 | # after the move to disk: a fleet of ~30 small replicas peaks around 150M, |
| 50 | # a single ~400M tenant database around 500M. Two earlier lessons are |
| 51 | # load-bearing and kept: the service was OOM-killed at a ~256M peak when |
| 52 | # restored copies ACCUMULATED (RestoreVerify now deletes each once checked |
| 53 | # — see its comment), and tmpfs is why unfitting files were fatal rather |
| 54 | # than slow. |
| 55 | # |
| 56 | # WATCH: a single database approaching this ceiling will throttle, not |
| 57 | # fail, when it crosses — raise this then. |
| 58 | MemoryMax=512M |
| 59 | |
amadan