amadan

carlos / platform Public

zdd plan 2 task 9: control service starts, stops and sweeps generation units; /parent

/start, /stop and /reset take gen=<n> for the run role. A generation start
writes the per-generation runtime drop-in (plan 2 D1), reloads, refuses the
start unless systemd applied the drop-in's slice, sets the host slice's
tier only when the slice is empty, then starts the unit. Every failure on
that path removes the drop-in and exe layout it wrote. /stop with kill=1
kills, stops and waits for the generation's cgroup; /sweep also lists and
cleans generation units; /parent sets the slice's memory to one tier or two
summed. Requests without gen= take today's path unchanged. Dormant: nothing
calls the gen verbs yet.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude cc299b5ee48f32d0303a450bece6b88317492638 parent 4475f63
7 files changed, +620 −19
  • internal/host/units/carlos-instance-control.service +3 −1
  • internal/host/units_isolation_test.go +1 −1
  • internal/instancectl/client.go +43 −0
  • internal/instancectl/gen.go +217 −0
  • internal/instancectl/hardening.go +6 −5
  • internal/instancectl/server.go +61 −12
  • internal/instancectl/server_gen_test.go +289 −0
diff --git a/internal/host/units/carlos-instance-control.service b/internal/host/units/carlos-instance-control.service
index b3eaab0ac..59b2d2b55 100644
--- a/internal/host/units/carlos-instance-control.service
+++ b/internal/host/units/carlos-instance-control.service
@@ -30,7 +30,9 @@ After=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount
[Service]
ExecStart=/opt/carlos/carlos instance-control
ProtectSystem=strict
-ReadWritePaths=/run/carlos-control
+# /run/systemd/system: the per-generation runtime drop-in (zero-downtime
+# deploys, plan 2 D1); nothing else is written there.
+ReadWritePaths=/run/carlos-control /run/systemd/system
ProtectHome=yes
PrivateTmp=yes
PrivateDevices=yes
diff --git a/internal/host/units_isolation_test.go b/internal/host/units_isolation_test.go
index c56c46235..b2d3336ee 100644
--- a/internal/host/units_isolation_test.go
+++ b/internal/host/units_isolation_test.go
@@ -322,7 +322,7 @@ func TestControlUnitsShape(t *testing.T) {
}
u := readUnit(t, "carlos-instance-control.service")
for _, want := range []string{"ExecStart=/opt/carlos/carlos instance-control", "Requires=carlos-instance-control.socket",
- "data-instances.mount", "ProtectSystem=strict", "ReadWritePaths=/run/carlos-control", "RestrictAddressFamilies=AF_UNIX", "MemoryMax=64M"} {
+ "data-instances.mount", "ProtectSystem=strict", "ReadWritePaths=/run/carlos-control /run/systemd/system", "RestrictAddressFamilies=AF_UNIX", "MemoryMax=64M"} {
if !strings.Contains(u, want) {
t.Errorf("control service lacks %q", want)
}
diff --git a/internal/instancectl/client.go b/internal/instancectl/client.go
index cae0ca4cb..0b7029001 100644
--- a/internal/instancectl/client.go
+++ b/internal/instancectl/client.go
@@ -170,6 +170,49 @@ func (c *Client) Reset(ctx context.Context, host string, role Role) error {
return err
}
+// genValues is the query of a generation verb: the run role and gen=<n>.
+func genValues(host string, n uint64) url.Values {
+ q := hostRole(host, RoleRun)
+ q.Set("gen", strconv.FormatUint(n, 10))
+ return q
+}
+
+// StartGen starts generation n's unit (zero-downtime spec §7.6) with its
+// tier and returns the invocation id the server read after the start.
+func (c *Client) StartGen(ctx context.Context, host string, n uint64, memoryMax int64) (string, error) {
+ q := genValues(host, n)
+ q.Set("memory_max", strconv.FormatInt(memoryMax, 10))
+ var out StartResponse
+ if _, err := c.do(ctx, DeadlineControl, http.MethodPost, "/start", q, &out); err != nil {
+ return "", err
+ }
+ return out.InvocationID, nil
+}
+
+// StopGen stops generation n's unit only. kill kills it outright and
+// returns once its cgroup is empty.
+func (c *Client) StopGen(ctx context.Context, host string, n uint64, kill bool) error {
+ q := genValues(host, n)
+ if kill {
+ q.Set("kill", "1")
+ }
+ _, err := c.do(ctx, DeadlineControl, http.MethodPost, "/stop", q, nil)
+ return err
+}
+
+func (c *Client) ResetGen(ctx context.Context, host string, n uint64) error {
+ _, err := c.do(ctx, DeadlineControl, http.MethodPost, "/reset", genValues(host, n), nil)
+ return err
+}
+
+// Parent sets host's slice memory, the generation parent's backstop (spec
+// §7.3): one tier, or two summed during an overlap.
+func (c *Client) Parent(ctx context.Context, host string, memoryMax int64) error {
+ q := url.Values{"host": {host}, "memory_max": {strconv.FormatInt(memoryMax, 10)}}
+ _, err := c.do(ctx, DeadlineControl, http.MethodPost, "/parent", q, nil)
+ return err
+}
+
// Sweep stops every instance unit on the box (design §6.7). A 502 carries
// the units that were stopped and the ones that would not; both are
// returned with the error.
diff --git a/internal/instancectl/gen.go b/internal/instancectl/gen.go
new file mode 100644
index 000000000..277aab297
--- /dev/null
+++ b/internal/instancectl/gen.go
@@ -0,0 +1,217 @@
+package instancectl
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/provision"
+)
+
+// Generation units (zero-downtime deploys spec §7.6, plan 2 D1). The
+// template carlos-instance-gen@<iid>:<n>.service cannot key its slice and
+// data mounts by itself: systemd has no specifier for part of an instance
+// string. So before each start this service writes a runtime drop-in with
+// everything the template leaves out, reloads, checks that systemd took
+// it, and only then starts the unit.
+
+// DropInName is the drop-in's file name inside <unit>.d.
+const DropInName = "50-carlos-generation.conf"
+
+// GenDropIn is the drop-in's text. The paths are the templates' literal
+// ones (/data/instances, /run/carlos/instances, /run/carlos-control/exe).
+// The memory lines are the generation's own tier (spec §7.3). The
+// template's OOMPolicy=kill gives the unit's cgroup memory.oom.group=1.
+func GenDropIn(host string, n uint64, tier cgroup.Limit) string {
+ iid := provision.InstanceID(host)
+ key := GenKey(host, n)
+ max := strconv.FormatInt(tier.MemoryMax, 10)
+ return "# Written by carlos instance-control for one generation; removed when it stops (zero-downtime plan 2).\n" +
+ "[Service]\n" +
+ "Slice=" + SliceName(host) + "\n" +
+ "BindPaths=/data/instances/" + iid + " /run/carlos/instances/" + iid + "\n" +
+ "ReadWritePaths=/data/instances/" + iid + " /run/carlos/instances/" + iid + "\n" +
+ "BindReadOnlyPaths=/run/carlos-control/exe/" + key + "/app:/opt/carlos/app /run/carlos-control/exe/" + key + "/live:/opt/carlos/live\n" +
+ "StandardOutput=file:" + GenLogSocket(host, n) + "\n" +
+ "MemoryMax=" + max + "\n" +
+ "MemoryHigh=" + strconv.FormatInt(cgroup.HighOf(tier.MemoryMax), 10) + "\n" +
+ "MemorySwapMax=" + max + "\n"
+}
+
+// writeGenDropIn writes the drop-in atomically: a temp file systemd never
+// reads (only *.conf is read), renamed into place.
+func (s *Server) writeGenDropIn(host string, n uint64, tier cgroup.Limit) error {
+ dir := filepath.Join(s.DropInDir, GenUnitName(host, n)+".d")
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ return err
+ }
+ tmp := filepath.Join(dir, "."+DropInName+".tmp")
+ if err := os.WriteFile(tmp, []byte(GenDropIn(host, n, tier)), 0o644); err != nil {
+ return err
+ }
+ return os.Rename(tmp, filepath.Join(dir, DropInName))
+}
+
+// ensureGenExe lays out generation n's view of its build under ExeRoot,
+// keyed by GenKey, as ensureHostExe does for the original template. Its
+// target is the host's live link, as today. Plan 3 binds the attempt's
+// staged path instead (spec §7.5 step 3).
+func (s *Server) ensureGenExe(host string, n uint64) error {
+ return ensureExe(s.ExeRoot, GenKey(host, n), filepath.Join(LiveDir, host), host)
+}
+
+// removeGenFiles removes a generation unit's drop-in and executable layout,
+// by the unit's name as systemd reports it. Best-effort: a leftover is a
+// few bytes in /run, never a running unit.
+func (s *Server) removeGenFiles(unit string) {
+ esc, n, ok := ParseGenUnitName(unit)
+ if !ok {
+ return
+ }
+ if s.DropInDir != "" {
+ _ = os.RemoveAll(filepath.Join(s.DropInDir, unit+".d"))
+ }
+ iid, ok := Unescape(esc)
+ if !ok || iid == "" || strings.ContainsAny(iid, "/\x00") || iid == "." || iid == ".." {
+ return
+ }
+ if s.ExeRoot != "" {
+ _ = os.RemoveAll(filepath.Join(s.ExeRoot, GenKeyOf(iid, n)))
+ }
+}
+
+// slicePopulated reports whether anything runs in host's slice. Absent is
+// no. Unreadable is yes, so a raised limit is never lowered on a guess.
+func (s *Server) slicePopulated(host string) bool {
+ b, err := os.ReadFile(filepath.Join(s.CgroupRoot, parentSlices, SliceName(host), "cgroup.events"))
+ if errors.Is(err, os.ErrNotExist) {
+ return false
+ }
+ if err != nil {
+ return true
+ }
+ return !strings.Contains(string(b), "populated 0")
+}
+
+// startGen is /start with a generation. The host slice is the parent (spec
+// §7.3): a cold start sets it to the tier. A start that finds the slice
+// populated leaves it alone, because a swap raised it with /parent before
+// the successor's start.
+func (s *Server) startGen(w http.ResponseWriter, r *http.Request, req request) {
+ ctx := r.Context()
+ unit := GenUnitName(req.host, req.gen)
+ if s.DropInDir == "" {
+ http.Error(w, "not starting "+unit+": generation units are not configured on this service", http.StatusNotImplemented)
+ return
+ }
+ // A refusal for the disk writes nothing. An exe layout that fails
+ // half-way may have made its directory, so that failure cleans up too.
+ if s.refuseStart(w, req.host, unit, func() error {
+ err := s.ensureGenExe(req.host, req.gen)
+ if err != nil {
+ s.removeGenFiles(unit)
+ }
+ return err
+ }) {
+ return
+ }
+ fail := func(status int, msg string) {
+ s.removeGenFiles(unit)
+ http.Error(w, msg, status)
+ }
+ if err := s.writeGenDropIn(req.host, req.gen, req.tier); err != nil {
+ fail(http.StatusBadGateway, "drop-in: "+err.Error())
+ return
+ }
+ if out, err := s.Systemctl(ctx, "daemon-reload"); err != nil {
+ fail(http.StatusBadGateway, "daemon-reload: "+strings.TrimSpace(out+" "+err.Error()))
+ return
+ }
+ if got := s.showValue(ctx, "Slice", unit); got != SliceName(req.host) {
+ fail(http.StatusBadGateway, "not starting "+unit+": its drop-in did not apply (slice "+strconv.Quote(got)+")")
+ return
+ }
+ if !s.slicePopulated(req.host) {
+ if err := s.setTier(ctx, req.host, req.tier); err != nil {
+ fail(http.StatusBadGateway, "set-property: "+err.Error())
+ return
+ }
+ }
+ if out, err := s.Systemctl(ctx, "start", unit); err != nil {
+ // A start that failed leaves a failed or half-started unit and the
+ // files it was given. Stop it (on a context the client's hang-up
+ // cannot cancel) and take the files away, so the next attempt
+ // begins from nothing.
+ dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+ defer cancel()
+ s.stopUnit(dctx, unit)
+ fail(http.StatusBadGateway, "start: "+strings.TrimSpace(out+" "+err.Error()))
+ return
+ }
+ inv, err := s.Systemctl(ctx, "show", "-p", "InvocationID", "--value", unit)
+ if err != nil {
+ http.Error(w, "show: "+err.Error(), http.StatusBadGateway)
+ return
+ }
+ s.logf("instance-control: %s: started %s memory.max=%d", req.host, unit, req.tier.MemoryMax)
+ writeJSON(w, http.StatusOK, StartResponse{InvocationID: strings.TrimSpace(inv)})
+}
+
+// stopGen is /stop with a generation. With kill=1 the generation is killed
+// outright (spec §8.2 step 1) and waited on until its cgroup is empty.
+// Either way the stop acts on that generation's unit only, and its files
+// go once it has stopped.
+func (s *Server) stopGen(w http.ResponseWriter, r *http.Request, req request) {
+ ctx := r.Context()
+ unit := GenUnitName(req.host, req.gen)
+ if req.kill {
+ if out, err := s.Systemctl(ctx, "kill", "--signal=SIGKILL", unit); err != nil && !strings.Contains(out+err.Error(), "not loaded") {
+ s.logf("instance-control: %s: kill %s: %v", req.host, unit, err)
+ }
+ }
+ if out, err := s.Systemctl(ctx, "stop", unit); err != nil {
+ http.Error(w, "stop: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway)
+ return
+ }
+ if req.kill {
+ wctx, cancel := context.WithTimeout(ctx, s.waitBudget())
+ defer cancel()
+ if err := WaitCgroupEmpty(wctx, s.CgroupRoot, SliceName(req.host), unit, 50*time.Millisecond); err != nil {
+ http.Error(w, "stop: "+err.Error(), http.StatusBadGateway)
+ return
+ }
+ }
+ s.removeGenFiles(unit)
+ w.WriteHeader(http.StatusNoContent)
+}
+
+// handleParent sets the host slice's memory: the generation parent's
+// backstop (spec §7.3). The swap raises it before the successor starts and
+// restores it once the retiring generation has stopped. CPU and IO shares
+// stay at one tier (plan 2 D6).
+func (s *Server) handleParent(w http.ResponseWriter, r *http.Request) {
+ q := r.URL.Query()
+ host := q.Get("host")
+ if !provision.ValidHostShape(host) {
+ http.Error(w, "host "+strconv.Quote(host)+" is not a host", http.StatusBadRequest)
+ return
+ }
+ n, err := strconv.ParseInt(q.Get("memory_max"), 10, 64)
+ if err != nil || !cgroup.ValidParentLimit(cgroup.Limit{MemoryMax: n}) {
+ http.Error(w, "memory_max "+strconv.Quote(q.Get("memory_max"))+" is not one tier or two", http.StatusBadRequest)
+ return
+ }
+ max := strconv.FormatInt(n, 10)
+ if out, err := s.Systemctl(r.Context(), "set-property", "--runtime", SliceName(host),
+ "MemoryMax="+max, "MemoryHigh="+strconv.FormatInt(cgroup.HighOf(n), 10), "MemorySwapMax="+max); err != nil {
+ http.Error(w, "set-property: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway)
+ return
+ }
+ w.WriteHeader(http.StatusNoContent)
+}
diff --git a/internal/instancectl/hardening.go b/internal/instancectl/hardening.go
index 2653c9902..e141cdada 100644
--- a/internal/instancectl/hardening.go
+++ b/internal/instancectl/hardening.go
@@ -314,9 +314,10 @@ func (s *Server) diskRoom() error {
return nil
}
-// refuseStart answers a start the disk or the exe layout cannot take. It
-// reports whether it answered.
-func (s *Server) refuseStart(w http.ResponseWriter, host, unit string) bool {
+// refuseStart answers a start the disk or the exe layout cannot take. exe
+// lays out the unit's view of its build (ensureHostExe, or ensureGenExe
+// for a generation). It reports whether it answered.
+func (s *Server) refuseStart(w http.ResponseWriter, host, unit string, exe func() error) bool {
if err := s.diskRoom(); err != nil {
s.logf("instance-control: %s: not starting %s: %v", host, unit, err)
status := http.StatusBadGateway
@@ -326,8 +327,8 @@ func (s *Server) refuseStart(w http.ResponseWriter, host, unit string) bool {
http.Error(w, "not starting "+unit+": "+err.Error(), status)
return true
}
- if s.ExeRoot != "" {
- if err := s.ensureHostExe(host); err != nil {
+ if s.ExeRoot != "" && exe != nil {
+ if err := exe(); err != nil {
http.Error(w, "exe: "+err.Error(), http.StatusBadGateway)
return true
}
diff --git a/internal/instancectl/server.go b/internal/instancectl/server.go
index 64b96d55c..ac0b38f29 100644
--- a/internal/instancectl/server.go
+++ b/internal/instancectl/server.go
@@ -57,6 +57,10 @@ type Server struct {
// ExeRoot is where ensureExe lays out each instance's view of its own
// build (#214 item 2); empty skips it, for tests of other verbs.
ExeRoot string
+ // DropInDir is where generation units' runtime drop-ins go
+ // (/run/systemd/system). Empty refuses every generation start: the
+ // template alone runs nothing (zero-downtime plan 2 D1).
+ DropInDir string
// ResolvConf is the box's resolv.conf and ResolvedConf resolved's
// upstream list; whichever an instance sees (resolverFile) has its
// nameservers re-opened through the egress fence before every probe
@@ -138,6 +142,7 @@ func runWithClaim(args []string, claim http.Handler) {
ProbeEvery: 5 * time.Minute,
Logf: log.Printf,
ExeRoot: filepath.Join(controlsock.Dir, "exe"),
+ DropInDir: "/run/systemd/system",
ResolvConf: "/etc/resolv.conf",
ResolvedConf: ResolvedConf,
LocalAddrs: func() ([]netip.Prefix, error) { return localPrefixes(fibTriePath, ifInet6Path) },
@@ -258,6 +263,7 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("POST /stop", s.claimAwareLegacy(s.handleStop))
mux.HandleFunc("POST /reset", s.claimAwareLegacy(s.handleReset))
mux.HandleFunc("POST /sweep", s.claimAwareLegacy(s.handleSweep))
+ mux.HandleFunc("POST /parent", s.claimAwareLegacy(s.handleParent))
mux.HandleFunc("GET /ready", s.handleReady)
return mux
}
@@ -268,22 +274,32 @@ type request struct {
host string
role Role
tier cgroup.Limit
+ gen uint64 // a generation of the run role (zero-downtime spec §7.6); 0 is none
+ kill bool
}
-func parseRequest(r *http.Request, roles []Role, wantTier bool) (request, error) {
+// verbShape is what a verb's query may carry.
+type verbShape struct {
+ roles []Role
+ tier bool
+ gen bool
+ kill bool
+}
+
+func parseRequest(r *http.Request, v verbShape) (request, error) {
q := r.URL.Query()
host := q.Get("host")
if !provision.ValidHostShape(host) {
return request{}, fmt.Errorf("host %q is not a host", host)
}
req := request{host: host}
- if len(roles) > 0 {
+ if len(v.roles) > 0 {
role, ok := ParseRole(q.Get("role"))
if !ok {
return request{}, fmt.Errorf("role %q is not run, seed or sidecar", q.Get("role"))
}
allowed := false
- for _, r := range roles {
+ for _, r := range v.roles {
allowed = allowed || r == role
}
if !allowed {
@@ -291,7 +307,7 @@ func parseRequest(r *http.Request, roles []Role, wantTier bool) (request, error)
}
req.role = role
}
- if wantTier {
+ if v.tier {
n, err := strconv.ParseInt(q.Get("memory_max"), 10, 64)
if err != nil {
return request{}, fmt.Errorf("memory_max %q is not a number", q.Get("memory_max"))
@@ -303,6 +319,26 @@ func parseRequest(r *http.Request, roles []Role, wantTier bool) (request, error)
return request{}, fmt.Errorf("memory_max %d is not a platform tier", n)
}
}
+ if q.Has("gen") {
+ raw := q.Get("gen")
+ if !v.gen {
+ return request{}, errors.New("this verb takes no generation")
+ }
+ n, err := strconv.ParseUint(raw, 10, 64)
+ if err != nil || n == 0 || strconv.FormatUint(n, 10) != raw {
+ return request{}, fmt.Errorf("gen %q is not a generation", raw)
+ }
+ if req.role != RoleRun {
+ return request{}, fmt.Errorf("role %s has no generations", req.role)
+ }
+ req.gen = n
+ }
+ if q.Has("kill") {
+ if !v.kill || req.gen == 0 || q.Get("kill") != "1" {
+ return request{}, errors.New("kill=1 belongs to a generation stop only")
+ }
+ req.kill = true
+ }
return req, nil
}
@@ -327,14 +363,18 @@ func writeJSON(w http.ResponseWriter, status int, v any) {
}
func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) {
- req, err := parseRequest(r, []Role{RoleRun, RoleSidecar}, true)
+ req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSidecar}, tier: true, gen: true})
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
+ if req.gen != 0 {
+ s.startGen(w, r, req)
+ return
+ }
ctx := r.Context()
unit := UnitName(req.role, req.host)
- if s.refuseStart(w, req.host, unit) {
+ if s.refuseStart(w, req.host, unit, func() error { return s.ensureHostExe(req.host) }) {
return
}
if err := s.setTier(ctx, req.host, req.tier); err != nil {
@@ -388,14 +428,14 @@ func parseShow(out string) map[string]string {
// to empty: a populated cgroup after the wait is still reported as a 502
// naming the unit.
func (s *Server) handleSeed(w http.ResponseWriter, r *http.Request) {
- req, err := parseRequest(r, nil, true)
+ req, err := parseRequest(r, verbShape{tier: true})
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
ctx := r.Context()
unit := UnitName(RoleSeed, req.host)
- if s.refuseStart(w, req.host, unit) {
+ if s.refuseStart(w, req.host, unit, func() error { return s.ensureHostExe(req.host) }) {
return
}
if err := s.setTier(ctx, req.host, req.tier); err != nil {
@@ -471,11 +511,15 @@ func (s *Server) stopUnit(ctx context.Context, unit string) {
}
func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) {
- req, err := parseRequest(r, []Role{RoleRun, RoleSeed, RoleSidecar}, false)
+ req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSeed, RoleSidecar}, gen: true, kill: true})
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
+ if req.gen != 0 {
+ s.stopGen(w, r, req)
+ return
+ }
if out, err := s.Systemctl(r.Context(), "stop", UnitName(req.role, req.host)); err != nil {
http.Error(w, "stop: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway)
return
@@ -484,12 +528,16 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) handleReset(w http.ResponseWriter, r *http.Request) {
- req, err := parseRequest(r, []Role{RoleRun, RoleSeed, RoleSidecar}, false)
+ req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSeed, RoleSidecar}, gen: true})
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
- if out, err := s.Systemctl(r.Context(), "reset-failed", UnitName(req.role, req.host)); err != nil && !strings.Contains(out+err.Error(), "not loaded") {
+ unit := UnitName(req.role, req.host)
+ if req.gen != 0 {
+ unit = GenUnitName(req.host, req.gen)
+ }
+ if out, err := s.Systemctl(r.Context(), "reset-failed", unit); err != nil && !strings.Contains(out+err.Error(), "not loaded") {
http.Error(w, "reset-failed: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway)
return
}
@@ -504,7 +552,7 @@ func (s *Server) handleSweep(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
targets := map[string]bool{}
out, err := s.Systemctl(ctx, "list-units", "--all", "--plain", "--no-legend", "--no-pager",
- UnitPrefixRun+"*.service", UnitPrefixSeed+"*.service", UnitPrefixSidecar+"*.service")
+ UnitPrefixRun+"*.service", UnitPrefixSeed+"*.service", UnitPrefixSidecar+"*.service", UnitPrefixGen+"*.service")
if err != nil {
http.Error(w, "list-units: "+err.Error(), http.StatusBadGateway)
return
@@ -565,6 +613,7 @@ func (s *Server) handleSweep(w http.ResponseWriter, r *http.Request) {
continue
}
stopped = append(stopped, unit)
+ s.removeGenFiles(unit)
}
if len(failed) > 0 {
writeJSON(w, http.StatusBadGateway, SweepResponse{Units: stopped, Failed: failed})
diff --git a/internal/instancectl/server_gen_test.go b/internal/instancectl/server_gen_test.go
new file mode 100644
index 000000000..0f93b13fb
--- /dev/null
+++ b/internal/instancectl/server_gen_test.go
@@ -0,0 +1,289 @@
+package instancectl
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/cgroup"
+)
+
+const genHost = "bill-2.example.com"
+
+var genUnit = GenUnitName(genHost, 7)
+
+func genServer(t *testing.T, ctl *fakeCtl) (*Server, string, string) {
+ t.Helper()
+ dropins, exe := t.TempDir(), t.TempDir()
+ s, sock := serveOnSocket(t, ctl, func(s *Server) { s.DropInDir = dropins; s.ExeRoot = exe })
+ return s, sock, dropins
+}
+
+func genCtl() *fakeCtl {
+ return &fakeCtl{answers: map[string]string{
+ "show -p Slice --value": SliceName(genHost) + "\n",
+ "show -p InvocationID --value": "abc123\n",
+ "list-units": "",
+ "list-jobs": "",
+ "show -p ActiveState --value": "inactive\n",
+ }}
+}
+
+func TestGenDropInText(t *testing.T) {
+ want := `# Written by carlos instance-control for one generation; removed when it stops (zero-downtime plan 2).
+[Service]
+Slice=carlos-instances-bill\x2d2.example.com.slice
+BindPaths=/data/instances/bill-2.example.com /run/carlos/instances/bill-2.example.com
+ReadWritePaths=/data/instances/bill-2.example.com /run/carlos/instances/bill-2.example.com
+BindReadOnlyPaths=/run/carlos-control/exe/bill-2.example.com@7/app:/opt/carlos/app /run/carlos-control/exe/bill-2.example.com@7/live:/opt/carlos/live
+StandardOutput=file:/run/carlos/logs/bill-2.example.com@7.run.sock
+MemoryMax=536870912
+MemoryHigh=483183819
+MemorySwapMax=536870912
+`
+ if got := GenDropIn(genHost, 7, cgroup.TierDefault); got != want {
+ t.Errorf("GenDropIn =\n%s\nwant\n%s", got, want)
+ }
+}
+
+func TestGenStartWritesTheDropInReloadsChecksTheSliceThenStarts(t *testing.T) {
+ ctl := genCtl()
+ s, sock, dropins := genServer(t, ctl)
+ inv, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax)
+ if err != nil || inv != "abc123" {
+ t.Fatalf("StartGen = %q, %v", inv, err)
+ }
+ raw, err := os.ReadFile(filepath.Join(dropins, genUnit+".d", DropInName))
+ if err != nil || string(raw) != GenDropIn(genHost, 7, cgroup.TierDefault) {
+ t.Fatalf("drop-in = %q, %v", raw, err)
+ }
+ if target, err := os.Readlink(filepath.Join(s.ExeRoot, GenKey(genHost, 7), "app")); err != nil || target != "/opt/carlos/live/"+genHost {
+ t.Fatalf("exe app link = %q, %v", target, err)
+ }
+ want := strings.Join([]string{
+ "daemon-reload",
+ "show -p Slice --value " + genUnit,
+ `set-property --runtime carlos-instances-bill\x2d2.example.com.slice MemoryMax=536870912 MemoryHigh=483183819 MemorySwapMax=536870912 CPUWeight=100 CPUQuota=100% IOWeight=100`,
+ "start " + genUnit,
+ "show -p InvocationID --value " + genUnit,
+ }, "\n")
+ if got := ctl.joined(); got != want {
+ t.Errorf("calls =\n%s\nwant\n%s", got, want)
+ }
+}
+
+// A successor's start during an overlap: the slice holds the incumbent,
+// and its raised limit is the swap's. The start leaves it alone.
+func TestGenStartLeavesAPopulatedSliceAlone(t *testing.T) {
+ ctl := genCtl()
+ s, sock, _ := genServer(t, ctl)
+ events := filepath.Join(s.CgroupRoot, parentSlices, SliceName(genHost), "cgroup.events")
+ if err := os.MkdirAll(filepath.Dir(events), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(events, []byte("populated 1\nfrozen 0\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(ctl.joined(), "set-property") {
+ t.Errorf("a successor's start touched the live slice:\n%s", ctl.joined())
+ }
+}
+
+// Review focus 1, nested: an empty slice is reset to the tier.
+func TestGenStartResetsAnEmptySlice(t *testing.T) {
+ ctl := genCtl()
+ s, sock, _ := genServer(t, ctl)
+ events := filepath.Join(s.CgroupRoot, parentSlices, SliceName(genHost), "cgroup.events")
+ if err := os.MkdirAll(filepath.Dir(events), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(events, []byte("populated 0\nfrozen 0\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(ctl.joined(), "set-property --runtime "+SliceName(genHost)+" MemoryMax=536870912") {
+ t.Errorf("a cold start left the slice as it was:\n%s", ctl.joined())
+ }
+}
+
+// Review focus 5: a drop-in systemd did not apply means the default slice
+// and an empty /data. Refuse, and leave nothing behind.
+func TestGenStartRefusesWhenTheDropInDidNotApply(t *testing.T) {
+ ctl := genCtl()
+ ctl.answers["show -p Slice --value"] = `system-carlos\x2dinstance\x2dgen.slice` + "\n"
+ s, sock, dropins := genServer(t, ctl)
+ if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err == nil {
+ t.Fatal("started a generation whose drop-in did not apply")
+ }
+ if strings.Contains(ctl.joined(), "start "+genUnit) {
+ t.Errorf("the unit was started:\n%s", ctl.joined())
+ }
+ if _, err := os.Stat(filepath.Join(dropins, genUnit+".d")); !os.IsNotExist(err) {
+ t.Errorf("drop-in left behind: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
+ t.Errorf("exe dir left behind: %v", err)
+ }
+}
+
+func TestGenStartRefusedWithoutADropInDir(t *testing.T) {
+ ctl := genCtl()
+ _, sock := serveOnSocket(t, ctl)
+ if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err == nil || !strings.Contains(err.Error(), "501") {
+ t.Fatalf("StartGen without DropInDir: %v, want a 501", err)
+ }
+ if ctl.joined() != "" {
+ t.Errorf("systemctl was called:\n%s", ctl.joined())
+ }
+}
+
+func TestGenParametersRefusedWhereTheyDoNotBelong(t *testing.T) {
+ ctl := genCtl()
+ _, c := newServer(t, ctl, func(s *Server) { s.DropInDir = t.TempDir() })
+ for _, path := range []string{
+ "/start?host=" + genHost + "&role=sidecar&memory_max=536870912&gen=7",
+ "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=0",
+ "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=07",
+ "/seed?host=" + genHost + "&memory_max=536870912&gen=7",
+ "/stop?host=" + genHost + "&role=run&kill=1",
+ "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=7&kill=1",
+ } {
+ if resp := post(t, c, path); resp.StatusCode != 400 {
+ t.Errorf("%s: status %d, want 400", path, resp.StatusCode)
+ }
+ }
+ if ctl.joined() != "" {
+ t.Errorf("systemctl was called:\n%s", ctl.joined())
+ }
+}
+
+func TestGenStopWithKillKillsStopsWaitsAndRemovesFiles(t *testing.T) {
+ ctl := genCtl()
+ s, sock, dropins := genServer(t, ctl)
+ if err := s.writeGenDropIn(genHost, 7, cgroup.TierDefault); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.ensureGenExe(genHost, 7); err != nil {
+ t.Fatal(err)
+ }
+ if err := NewClient(sock).StopGen(context.Background(), genHost, 7, true); err != nil {
+ t.Fatal(err)
+ }
+ if want := "kill --signal=SIGKILL " + genUnit + "\nstop " + genUnit; ctl.joined() != want {
+ t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want)
+ }
+ if _, err := os.Stat(filepath.Join(dropins, genUnit+".d")); !os.IsNotExist(err) {
+ t.Errorf("drop-in survived the stop: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
+ t.Errorf("exe dir survived the stop: %v", err)
+ }
+}
+
+func TestParentSetsOnlyTheSliceMemory(t *testing.T) {
+ ctl := genCtl()
+ _, sock := serveOnSocket(t, ctl)
+ c := NewClient(sock)
+ if err := c.Parent(context.Background(), genHost, cgroup.OverlapLimit(cgroup.TierDefault, cgroup.TierDefault).MemoryMax); err != nil {
+ t.Fatal(err)
+ }
+ want := `set-property --runtime carlos-instances-bill\x2d2.example.com.slice MemoryMax=1073741824 MemoryHigh=966367638 MemorySwapMax=1073741824`
+ if ctl.joined() != want {
+ t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want)
+ }
+ if err := c.Parent(context.Background(), genHost, 64<<20); err == nil {
+ t.Error("a parent limit that is no tier sum was accepted")
+ }
+}
+
+func TestSweepStopsGenerationUnitsAndRemovesTheirFiles(t *testing.T) {
+ ctl := genCtl()
+ ctl.answers["list-units"] = genUnit + " loaded active running carlos exec-backed instance generation\n"
+ s, sock, dropins := genServer(t, ctl)
+ if err := s.writeGenDropIn(genHost, 7, cgroup.TierDefault); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.ensureGenExe(genHost, 7); err != nil {
+ t.Fatal(err)
+ }
+ out, err := NewClient(sock).Sweep(context.Background())
+ if err != nil || len(out.Units) != 1 || out.Units[0] != genUnit {
+ raw, _ := json.Marshal(out)
+ t.Fatalf("sweep = %s, %v", raw, err)
+ }
+ if !strings.Contains(ctl.joined(), UnitPrefixGen+"*.service") {
+ t.Errorf("list-units did not ask for generation units:\n%s", ctl.joined())
+ }
+ if _, err := os.Stat(filepath.Join(dropins, genUnit+".d")); !os.IsNotExist(err) {
+ t.Errorf("drop-in survived the sweep: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
+ t.Errorf("exe dir survived the sweep: %v", err)
+ }
+}
+
+// A start that failed leaves nothing behind: the unit is stopped and the
+// drop-in and exe layout are removed.
+func TestGenStartFailureStopsTheUnitAndRemovesTheFiles(t *testing.T) {
+ ctl := genCtl()
+ ctl.fail = map[string]error{"start ": errors.New("boom")}
+ s, sock, dropins := genServer(t, ctl)
+ if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err == nil {
+ t.Fatal("a failed start was reported as started")
+ }
+ if !strings.Contains(ctl.joined(), "stop "+genUnit) {
+ t.Errorf("the failed unit was not stopped:\n%s", ctl.joined())
+ }
+ if _, err := os.Stat(filepath.Join(dropins, genUnit+".d")); !os.IsNotExist(err) {
+ t.Errorf("drop-in left behind: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
+ t.Errorf("exe dir left behind: %v", err)
+ }
+}
+
+func TestGenResetResetsTheGenerationUnitOnly(t *testing.T) {
+ ctl := genCtl()
+ _, sock := serveOnSocket(t, ctl)
+ if err := NewClient(sock).ResetGen(context.Background(), genHost, 7); err != nil {
+ t.Fatal(err)
+ }
+ if want := "reset-failed " + genUnit; ctl.joined() != want {
+ t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want)
+ }
+}
+
+func TestGenAndParentInputsAreValidated(t *testing.T) {
+ ctl := genCtl()
+ _, c := newServer(t, ctl, func(s *Server) { s.DropInDir = t.TempDir() })
+ for _, path := range []string{
+ "/start?host=" + genHost + "&role=run&memory_max=12345&gen=7",
+ "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=",
+ "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=-1",
+ "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=18446744073709551616",
+ "/stop?host=" + genHost + "&role=run&gen=7&kill=true",
+ "/stop?host=" + genHost + "&role=seed&gen=7",
+ "/reset?host=" + genHost + "&role=run&gen=0",
+ "/reset?host=" + genHost + "&role=sidecar&gen=7",
+ "/parent?host=not_a_host&memory_max=536870912",
+ "/parent?host=" + genHost,
+ "/parent?host=" + genHost + "&memory_max=-536870912",
+ "/parent?host=" + genHost + "&memory_max=0",
+ } {
+ if resp := post(t, c, path); resp.StatusCode != 400 {
+ t.Errorf("%s: status %d, want 400", path, resp.StatusCode)
+ }
+ }
+ if ctl.joined() != "" {
+ t.Errorf("systemctl was called:\n%s", ctl.joined())
+ }
+}