amadan

carlos / platform Public

claim-prev-rehearsal: the activation witness starts from a box the platform made (ruling R2)

TestRootActivationConsoleRuntimePayload now installs v0.9.0 through the
region-host README's four steps (binary, host.env, carlos bootstrap, the
edge), lets the real agent and carlos system apply update it to v1.0.0
so carlos.prev holds the release before, publishes the box record with
box publish-record, builds the roster with the real claims run phases,
reproduces the trial's 6.2 stop, then runs install-channel --policy,
guard-arm in leo's and sophie's forms, install-console, the guarded
console and step 7, with the agent's own box reports as evidence. No
hand-written carlos.prev and no hand-published box state. A third
embedded image (carlos v0.9.0) is built by activationImagesBuild.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude b9dcae2dc4df48ae6226a9178683754f669908e6 parent 5d1c3be
2 files changed, +631 −228
  • internal/host/activation_console_binary_rootrunner_linux_test.go +4 −0
  • internal/host/activation_console_rootrunner_linux_test.go +627 −228
diff --git a/internal/host/activation_console_binary_rootrunner_linux_test.go b/internal/host/activation_console_binary_rootrunner_linux_test.go
index 84d0b8407..0a8bba330 100644
--- a/internal/host/activation_console_binary_rootrunner_linux_test.go
+++ b/internal/host/activation_console_binary_rootrunner_linux_test.go
@@ -10,7 +10,11 @@ var activationConsoleBinary []byte
//go:embed activation-carlos-runner.bin
var activationCarlosBinary []byte
+//go:embed activation-carlos-old-runner.bin
+var activationCarlosOldBinary []byte
+
func init() {
activationConsoleImage = activationConsoleBinary
activationCarlosImage = activationCarlosBinary
+ activationCarlosOldImage = activationCarlosOldBinary
}
diff --git a/internal/host/activation_console_rootrunner_linux_test.go b/internal/host/activation_console_rootrunner_linux_test.go
index 55383db44..39ebfe63e 100644
--- a/internal/host/activation_console_rootrunner_linux_test.go
+++ b/internal/host/activation_console_rootrunner_linux_test.go
@@ -5,8 +5,10 @@ package host
import (
"bytes"
"crypto/ed25519"
+ "crypto/rand"
"debug/buildinfo"
"encoding/base64"
+ "encoding/hex"
"encoding/json"
"encoding/xml"
"errors"
@@ -17,6 +19,7 @@ import (
"net/http/httptest"
"os"
"os/exec"
+ "os/user"
"path/filepath"
"reflect"
"strings"
@@ -36,18 +39,21 @@ import (
"github.com/carlosframework/platform/internal/status"
)
-var activationConsoleImage, activationCarlosImage []byte
+var activationConsoleImage, activationCarlosImage, activationCarlosOldImage []byte
-// activationImagesBuild is how the two embedded images are built, from the
+// activationImagesBuild is how the three embedded images are built, from the
// worktree root, before `go test -c -tags rootrunner,activationconsole`
-// (controller ruling C5). Both are stripped (-s -w), as release builds are:
+// (controller ruling C5). All are stripped (-s -w), as release builds are:
// an unstripped Go binary carries compressed DWARF sections, which the
-// bounded ELF pre-check refuses (F1). Both are stamped v1.0.0 because the identity
-// names that version: guard-arm compares the carlos build with it, and the
-// console's own floor report (Disk.Running) compares the console's build
-// with it, so publish-identity refuses a console stamped otherwise.
-// checkActivationImages asserts both images match this command.
+// bounded ELF pre-check refuses (F1). The release under test is v1.0.0:
+// guard-arm compares the carlos build with the identity, and the console's
+// own floor report (Disk.Running) compares the console's build with it, so
+// publish-identity refuses a console stamped otherwise. v0.9.0 is the older
+// release the box is first installed with, so the update leaves it in
+// carlos.prev (prev-rehearsal R2). checkActivationImages asserts all three
+// images match this command.
const activationImagesBuild = `GOOS=linux GOARCH=arm64 CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -ldflags "-s -w -X github.com/carlosframework/platform/internal/version.Build=v1.0.0" -o internal/host/activation-carlos-runner.bin .
+GOOS=linux GOARCH=arm64 CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -ldflags "-s -w -X github.com/carlosframework/platform/internal/version.Build=v0.9.0" -o internal/host/activation-carlos-old-runner.bin .
GOOS=linux GOARCH=arm64 CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -ldflags "-s -w -X github.com/carlosframework/platform/internal/version.Build=v1.0.0" -o internal/host/activation-console-runner.bin ./cmd/console`
func checkActivationImages(t *testing.T) {
@@ -57,6 +63,7 @@ func checkActivationImages(t *testing.T) {
path, ldflag string
}{
{activationCarlosImage, "github.com/carlosframework/platform", "-s -w -X github.com/carlosframework/platform/internal/version.Build=v1.0.0"},
+ {activationCarlosOldImage, "github.com/carlosframework/platform", "-s -w -X github.com/carlosframework/platform/internal/version.Build=v0.9.0"},
{activationConsoleImage, "github.com/carlosframework/platform/cmd/console", "-s -w -X github.com/carlosframework/platform/internal/version.Build=v1.0.0"},
} {
info, err := buildinfo.Read(bytes.NewReader(img.raw))
@@ -73,25 +80,82 @@ func checkActivationImages(t *testing.T) {
}
}
-// This test allows only loopback HTTP to its S3 and console fixtures. The
-// executable, guard, systemd, proc observation, HTTP auth and reset are real.
-// Sophie uses the runner’s real AWS identity; Leo, box telemetry and initial
-// app/claim state are fixtures. No deployment bucket or tenant is touched.
+// activationInspected is `carlos system claims inspect`'s output.
+type activationInspected struct {
+ Roster hostactors.Roster `json:"roster"`
+ ETag string `json:"etag"`
+ SignedRoster json.RawMessage `json:"signed_roster"`
+}
+
+// activationPhaseResult is `carlos system claims run`'s output.
+type activationPhaseResult struct {
+ claimactivation.Result
+ Out string `json:"out,omitempty"`
+}
+
+// The claim trial's steps 6 and 7, rehearsed on a box the platform made
+// (prev-rehearsal ruling R2). Nothing on the box is written by hand but what
+// the region-host README's four steps write (the first binary and host.env)
+// and the box-only console.service with its database:
+//
+// - the box is installed at v0.9.0 (README steps 1 to 4: the binary,
+// host.env, `carlos bootstrap`, the edge), and the real agent and
+// `carlos system apply`, run by carlos-system-update.service, update it
+// to v1.0.0, leaving v0.9.0 in carlos.prev, as on the trial's boxes;
+// - its AWS identity record is published by `carlos system box
+// publish-record` as the agent user, as step 5.0 runs it;
+// - the roster is made by `claims run` initialize to enroll-console, and
+// the box and its console are enrolled with the runner's real AWS
+// identity;
+// - the trial's 6.2 stop is reproduced, then `install-channel --policy`
+// makes carlos.prev the release (R1), and step 6 and step 7 run as the
+// runbook gives them, with the real agent's box reports as evidence.
+//
+// One AWS instance can be enrolled as one box only (boxid refuses a second
+// reference for an instance), so this box is both the trial's hosting box
+// and its console box: guard-arm runs in leo's form, then in sophie's.
+// Fixtures that remain: the loopback S3 bucket, the console release (shipped
+// and promoted in-process with the production functions), publish-identity's
+// sums key (the vendor key cannot sign a fixture; the real binary's refusal
+// of the fixture sums is checked), the console's versions tree, the console
+// unit and database, and the claim state behind the authenticated reset.
+// Network: every carlos unit gets a runtime drop-in allowing only loopback
+// and the instance metadata service.
func TestRootActivationConsoleRuntimePayload(t *testing.T) {
- if os.Geteuid() != 0 || len(activationConsoleImage) == 0 || len(activationCarlosImage) == 0 {
+ if os.Geteuid() != 0 || len(activationConsoleImage) == 0 || len(activationCarlosImage) == 0 || len(activationCarlosOldImage) == 0 {
t.Fatal("requires root and embedded activation binaries")
}
checkActivationImages(t)
- uid, gid := generationRunnerRegistryUser(t)
+ if pid1, err := os.ReadFile("/proc/1/comm"); err != nil || strings.TrimSpace(string(pid1)) != "systemd" {
+ t.Fatal("root runner requires systemd", err)
+ }
instance := status.InstanceID()
if instance == "" {
t.Fatal("runner instance identity unavailable")
}
- for _, p := range []string{"/opt/carlos", "/etc/carlos", "/etc/systemd/system/console.service", "/etc/systemd/system/console.service.d", "/etc/systemd/system/carlos-edge.service.d", "/etc/systemd/system/carlos-system-update.service.d", "/etc/systemd/system/carlos-system-recover.service.d"} {
+ // The box's name is the one its agent publishes under.
+ box := status.Label(readMachineID())
+ if !boxid.ValidBox(box) {
+ t.Fatal("runner box name", box)
+ }
+ const bucket = "activation-fixture"
+ const consoleID = "dev-console"
+ opt := release.OptDir()
+ owned := []string{"/opt/carlos", "/etc/carlos", "/data", "/run/carlos", "/etc/systemd/system/carlos-edge.service", "/etc/systemd/system/console.service", "/etc/systemd/system/console.service.d"}
+ for _, p := range owned {
if _, err := os.Lstat(p); !os.IsNotExist(err) {
t.Fatal("pre-existing fixture path", p, err)
}
}
+ if _, err := user.Lookup(serviceUser); err == nil {
+ t.Fatal("the service user exists before the box is made")
+ }
+ // Everything bootstrap and the steps add is found by name afterwards.
+ listed := []string{"/etc/systemd/system", "/run/systemd/system", "/etc/tmpfiles.d", "/etc/sysusers.d", "/etc/systemd/journald.conf.d"}
+ before := map[string]map[string]bool{}
+ for _, dir := range listed {
+ before[dir] = activationDirNames(dir)
+ }
d := activationRootFixture(t)
check := func(err error) {
t.Helper()
@@ -103,110 +167,314 @@ func TestRootActivationConsoleRuntimePayload(t *testing.T) {
t.Helper()
check(os.MkdirAll(filepath.Dir(path), 0755))
check(os.WriteFile(path, b, mode))
+ check(os.Chmod(path, mode))
+ }
+ absent := func(path string) {
+ t.Helper()
+ if _, e := os.Lstat(path); !os.IsNotExist(e) {
+ t.Fatal("present", path, e)
+ }
+ }
+ journal := func(unit string, n int) string {
+ out, _ := exec.Command("journalctl", "-u", unit, "--no-pager", "-n", fmt.Sprint(n)).CombinedOutput()
+ return string(out)
}
t.Cleanup(func() {
- out, _ := exec.Command("journalctl", "-u", "console.service", "--no-pager", "-n", "45").CombinedOutput()
- t.Log(string(out))
- exec.Command("systemctl", "stop", "console.service").Run()
- for _, p := range []string{"/opt/carlos", "/etc/carlos", "/etc/systemd/system/console.service", "/etc/systemd/system/console.service.d", "/etc/systemd/system/carlos-edge.service.d", "/etc/systemd/system/carlos-system-update.service.d", "/etc/systemd/system/carlos-system-recover.service.d"} {
+ for _, u := range []string{"carlos-edge.service", "carlos-system-update.service", "console.service"} {
+ t.Log(u, "journal:\n"+journal(u, 80))
+ }
+ var added []string
+ for _, dir := range listed {
+ for name := range activationDirNames(dir) {
+ if !before[dir][name] {
+ added = append(added, filepath.Join(dir, name))
+ }
+ }
+ }
+ var unitNames []string
+ for _, p := range added {
+ if filepath.Dir(p) == "/etc/systemd/system" && !strings.HasSuffix(p, ".d") && !strings.HasSuffix(p, ".wants") {
+ unitNames = append(unitNames, filepath.Base(p))
+ }
+ }
+ exec.Command("systemctl", "stop", "console.service", "carlos-edge.service").Run()
+ if len(unitNames) > 0 {
+ exec.Command("systemctl", append([]string{"stop"}, unitNames...)...).Run()
+ exec.Command("systemctl", append([]string{"disable"}, unitNames...)...).Run()
+ }
+ for _, m := range []string{"data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount"} {
+ exec.Command("systemctl", "stop", m).Run()
+ }
+ for _, p := range added {
+ os.RemoveAll(p)
+ }
+ for _, p := range owned {
os.RemoveAll(p)
}
exec.Command("systemctl", "daemon-reload").Run()
- exec.Command("systemctl", "reset-failed", "console.service").Run()
+ exec.Command("systemctl", "reset-failed").Run()
+ exec.Command("userdel", serviceUser).Run()
+ exec.Command("groupdel", socketsGroup).Run()
})
store := release.NewDirStore(filepath.Join(d, "bucket"))
endpoint := activationS3Fixture(t, store)
pub, key, _ := ed25519.GenerateKey(nil)
signer := release.RootSigner(key)
- const bucket = "activation-fixture"
+ pubArg := base64.StdEncoding.EncodeToString(pub)
ctx := t.Context()
- directory := hostactors.Directory{Store: store, PublicKey: pub, Bucket: bucket}
- runner := claimactivation.Runner{Directory: directory, Signer: signer}
- phase := func(p claimactivation.Plan) {
- t.Helper()
- p.Bucket = bucket
- if o, e := directory.Read(ctx); e == nil {
- p.ExpectedRevision = o.Roster.Revision
+ // host.env (README step 2), with the hibernation block a hosting box
+ // carries, as leo's does. No release key: a box only verifies. (An agent
+ // with no CARLOS_HIBERNATE_BUCKET panics at start on this tree: runAgent
+ // dereferences the nil activator config at registerProductionMovesParks.
+ // Reported, not fixed here.)
+ hostEnv := []string{"CARLOS_DEPLOYMENT_BUCKET=" + bucket, "CARLOS_HIBERNATE_BUCKET=" + bucket + "-ie", "CARLOS_HIBERNATE_REGION=eu-west-1", "CARLOS_HIBERNATE_PREFIX=instances", "CARLOS_RELEASE_PUBKEY=" + pubArg, "AWS_REGION=eu-west-1", "AWS_DEFAULT_REGION=eu-west-1", "AWS_ACCESS_KEY_ID=fixture", "AWS_SECRET_ACCESS_KEY=fixture", "AWS_EC2_METADATA_DISABLED=true", "AWS_CONFIG_FILE=/nonexistent", "AWS_SHARED_CREDENTIALS_FILE=/nonexistent", "CARLOS_S3_ENDPOINT=" + endpoint}
+ for _, kv := range hostEnv {
+ if k, v, _ := strings.Cut(kv, "="); strings.HasPrefix(k, "AWS_") || k == "CARLOS_S3_ENDPOINT" {
+ t.Setenv(k, v)
}
- before := activationStoreSnapshot(t, store)
- preview, e := runner.Run(ctx, p, true)
- check(e)
- if !reflect.DeepEqual(before, activationStoreSnapshot(t, store)) {
- t.Fatal("dry run mutated bucket", p.Phase)
+ }
+ // A root shell on the box after `set -a; . /etc/carlos/host.env; set +a`,
+ // as the runbook's box blocks run. The signer's commands get the same
+ // bucket and credentials.
+ shellEnv := append([]string{"PATH=/usr/sbin:/usr/bin:/sbin:/bin"}, hostEnv...)
+ try := func(bin string, args ...string) (string, string, error) {
+ t.Helper()
+ cmd := exec.CommandContext(ctx, bin, args...)
+ cmd.Env = shellEnv
+ cmd.Dir = d
+ var stdout, stderr bytes.Buffer
+ cmd.Stdout, cmd.Stderr = &stdout, &stderr
+ err := cmd.Run()
+ return stdout.String(), stderr.String(), err
+ }
+ run := func(want bool, bin string, args ...string) (string, string) {
+ t.Helper()
+ stdout, stderr, err := try(bin, args...)
+ if (err == nil) != want {
+ t.Fatal(bin, args, "\nstdout:", stdout, "\nstderr:", stderr, err)
}
- result, e := runner.Run(ctx, p, false)
- check(e)
- if preview.Revision != result.Revision || len(preview.Writes) != len(result.Writes) {
- t.Fatal("preview/apply differ", p.Phase, preview, result)
+ return stdout, stderr
+ }
+ carlos := func(want bool, args ...string) (string, string) {
+ t.Helper()
+ return run(want, "/opt/carlos/carlos", args...)
+ }
+ ctl := func(args ...string) string {
+ t.Helper()
+ out, e := exec.CommandContext(ctx, "systemctl", args...).CombinedOutput()
+ if e != nil {
+ t.Fatal(args, string(out), e)
}
- t.Log("phase", p.Phase, "revision", result.Revision, "writes", len(result.Writes))
+ return strings.TrimSpace(string(out))
}
- phase(claimactivation.Plan{Phase: "initialize"})
- phase(claimactivation.Plan{Phase: "configure", Regions: map[string]hostactors.Region{"ie": {Account: "640012953448", AWSRegion: "eu-west-1", Namespaces: []string{"platform"}, Pools: []string{"console", "cohort1"}}}})
- identityRecord, err := readIdentityRecord(&http.Client{Timeout: 3 * time.Second}, imdsBase, "sophie", time.Now())
- check(err)
- check(boxid.Publish(ctx, store, identityRecord))
- boxPlan := claimactivation.BoxInput{Reference: "sophie", Namespace: "platform", Label: "sophie", Region: "ie", Pool: "console", AWSRegion: "eu-west-1", InstanceID: instance, Operator: "root-runner"}
- phase(claimactivation.Plan{Phase: "invite-box", Box: &boxPlan})
- phase(claimactivation.Plan{Phase: "enroll-box", Box: &boxPlan})
- o, err := directory.Read(ctx)
+ readers := func() (string, string) {
+ t.Helper()
+ cur, _ := run(true, systemBinaryPath(opt), "version")
+ prev, _ := run(true, systemPrevPath(opt), "version")
+ return strings.TrimSpace(cur), strings.TrimSpace(prev)
+ }
+
+ // The box, made by the platform's own install and update path.
+ // README step 1: the binary. The box's first build is the older release.
+ write(systemBinaryPath(opt), activationCarlosOldImage, 0755)
+ // Step 2: host.env.
+ write("/etc/carlos/host.env", []byte(strings.Join(hostEnv, "\n")+"\n"), 0600)
+ // Step 3: carlos bootstrap.
+ out, _ := carlos(true, "bootstrap")
+ t.Log("bootstrap:", out)
+ for _, u := range []string{"carlos-edge.service", "carlos-system-update.service", "carlos-system-recover.service", "carlos-adopt.service", "carlos-stage.service", "carlos-restart.service", "carlos-config-sync.service", "carlos-instance-control.service"} {
+ write("/run/systemd/system/"+u+".d/10-rootrunner-network.conf", []byte("[Service]\nIPAddressDeny=any\nIPAddressAllow=127.0.0.1 169.254.169.254\n"), 0644)
+ }
+ ctl("daemon-reload")
+ // The operator promotes v1.0.0 on the box's channel, as the console's
+ // system update door does.
+ artifact := release.SystemArtifact{Key: release.SystemArtifactKey("v1.0.0", "linux-arm64"), SHA256: hostfloor.Digest(activationCarlosImage)}
+ _, err := store.Put(ctx, artifact.Key, activationCarlosImage, "")
check(err)
- next := hostactors.Clone(o.Roster)
- next.Boxes["leo"] = hostactors.Box{Enrollment: strings.Repeat("2", 32), Namespace: "platform", BoxID: "leo", Label: "leo", Region: "ie", Pool: "cohort1", Cloud: hostactors.Cloud{Provider: "aws", Account: "640012953448", Region: "eu-west-1", InstanceID: "i-leo"}, EvidenceSHA256: strings.Repeat("b", 64), State: "active", LossIDs: []string{}}
- _, err = directory.Replace(ctx, signer, o, next)
+ _, err = release.PromoteSystem(ctx, store, signer, release.SystemComponentRouter, "stable", release.SystemRelease{Version: "v1.0.0", ReleaseTag: "v1.0.0", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{"linux-arm64": artifact}}, release.SystemPromoteOptions{Actor: "root-runner"})
check(err)
- c := hostactors.Console{InstanceID: "dev-console", Enrollment: strings.Repeat("c", 32), DeploymentBucket: bucket, Namespaces: []string{"platform"}, Compute: &hostactors.ComputeBinding{Box: "sophie", Enrollment: next.Boxes["sophie"].Enrollment}}
- phase(claimactivation.Plan{Phase: "invite-console", Console: &c})
- phase(claimactivation.Plan{Phase: "enroll-console", Console: &c})
- // The environment every real carlos command below runs with, as host.env
- // would give it on a box. In-process S3 clients get the same endpoint.
- environment := []string{"PATH=/usr/sbin:/usr/bin:/sbin:/bin", "CARLOS_DEPLOYMENT_BUCKET=" + bucket, "AWS_REGION=eu-west-1", "AWS_DEFAULT_REGION=eu-west-1", "AWS_ACCESS_KEY_ID=fixture", "AWS_SECRET_ACCESS_KEY=fixture", "AWS_EC2_METADATA_DISABLED=true", "AWS_CONFIG_FILE=/nonexistent", "AWS_SHARED_CREDENTIALS_FILE=/nonexistent", "CARLOS_S3_ENDPOINT=" + endpoint, "CARLOS_RELEASE_PUBKEY=" + base64.StdEncoding.EncodeToString(pub), "CARLOS_RELEASE_KEY=" + base64.StdEncoding.EncodeToString(key.Seed()), "CARLOS_DATA_DIR=" + d + "/data"}
- for _, kv := range environment {
- if k, v, _ := strings.Cut(kv, "="); strings.HasPrefix(k, "AWS_") || k == "CARLOS_S3_ENDPOINT" {
- t.Setenv(k, v)
+ // Step 4: the edge. Its agent finds the promote, fetches the build and
+ // asks root, and carlos-system-update.service applies it.
+ ctl("start", "carlos-edge.service")
+ deadline := time.Now().Add(5 * time.Minute)
+ for {
+ r, _ := readSystemResult(opt)
+ if r != nil && r.Version == "v1.0.0" && r.Outcome == systemOutcomeApplied {
+ break
+ }
+ if r != nil && r.Version == "v1.0.0" && r.Outcome != systemOutcomeDeferred {
+ t.Fatalf("the update to v1.0.0 ended %s: %s\n%s", r.Outcome, r.Reason, journal("carlos-system-update.service", 120))
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("the update to v1.0.0 did not apply: %+v\n%s\n%s", r, journal("carlos-edge.service", 120), journal("carlos-system-update.service", 120))
}
+ time.Sleep(2 * time.Second)
}
- pubArg := base64.StdEncoding.EncodeToString(pub)
- carlos := func(want bool, args ...string) string {
+ if j, _ := readSystemJournal(opt); j != nil {
+ t.Fatal("the update left its journal", j)
+ }
+ cur, _ := os.ReadFile(systemBinaryPath(opt))
+ prev, _ := os.ReadFile(systemPrevPath(opt))
+ if !bytes.Equal(cur, activationCarlosImage) || !bytes.Equal(prev, activationCarlosOldImage) {
+ t.Fatal("the update did not leave v1.0.0 in carlos and v0.9.0 in carlos.prev")
+ }
+ if applied, e := readSystemApplied(opt); e != nil || applied == nil || applied.Version != "v1.0.0" {
+ t.Fatal("applied", applied, e)
+ }
+ absent(channelIdentityPath(opt))
+ if c, p := readers(); c != "carlos v1.0.0" || p != "carlos v0.9.0" {
+ t.Fatal("readers before install-channel", c, p)
+ }
+ if state := ctl("is-active", "carlos-edge.service"); state != "active" {
+ t.Fatal("edge", state)
+ }
+ t.Log("box made: carlos v1.0.0, carlos.prev v0.9.0, by bootstrap and the real update")
+
+ // Step 5.0: the box's AWS identity record, as the agent user.
+ publishRecord := func() string {
+ t.Helper()
+ out, _ := run(true, "systemd-run", "--wait", "--pipe", "--quiet", "-p", "EnvironmentFile=/etc/carlos/host.env", "-p", "User="+serviceUser, "-p", "IPAddressDeny=any", "-p", "IPAddressAllow=127.0.0.1 169.254.169.254", systemBinaryPath(opt), "system", "box", "publish-record")
+ return strings.TrimSpace(out)
+ }
+ if line := publishRecord(); line != fmt.Sprintf("published the identity record for %s (instance %s) to control/boxes/%s.json", box, instance, box) {
+ t.Fatal("publish-record", line)
+ }
+ if line := publishRecord(); !strings.HasPrefix(line, fmt.Sprintf("the record at control/boxes/%s.json is valid AWS-signed evidence for %s (instance %s), so nothing was written", box, box, instance)) {
+ t.Fatal("publish-record verify", line)
+ }
+
+ // The trusted signing host: its CLI, the key in a root 0600 file.
+ sd := filepath.Join(d, "signer")
+ check(os.MkdirAll(sd, 0700))
+ write(sd+"/carlos", activationCarlosImage, 0755)
+ write(sd+"/release.key", []byte(base64.StdEncoding.EncodeToString(key.Seed())+"\n"), 0600)
+ signerCLI := func(want bool, args ...string) (string, string) {
+ t.Helper()
+ return run(want, sd+"/carlos", args...)
+ }
+ inspectWith := func(bin string) activationInspected {
+ t.Helper()
+ out, _ := run(true, bin, "system", "claims", "inspect", "--bucket", bucket, "--aws-region", "eu-west-1", "--public-key", pubArg)
+ var o activationInspected
+ check(json.Unmarshal([]byte(out), &o))
+ return o
+ }
+ refresh := func() activationInspected { t.Helper(); return inspectWith(sd + "/carlos") }
+ // The control objects a phase may write; the agent writes status beside
+ // them throughout.
+ controlSnapshot := func() map[string]string {
t.Helper()
- cmd := exec.CommandContext(ctx, "/opt/carlos/carlos", args...)
- cmd.Env = environment
- output, e := cmd.CombinedOutput()
- if (e == nil) != want {
- t.Fatal(args, string(output), e)
+ all := activationStoreSnapshot(t, store)
+ for k := range all {
+ keep := k == hostactors.Key || k == hostfloor.Key || k == hostfloor.AuthKey
+ for _, prefix := range []string{"control/host-actor-invitations/", "control/host-console-invitations/", "control/system/router/releases/"} {
+ keep = keep || strings.HasPrefix(k, prefix)
+ }
+ if !keep {
+ delete(all, k)
+ }
}
- return string(output)
+ return all
}
- absent := func(path string) {
+ runArgs := func(out string) []string {
+ args := []string{"system", "claims", "run", "--plan", sd + "/plan.json", "--signing-key", sd + "/release.key", "--public-key", pubArg, "--aws-region", "eu-west-1"}
+ if out != "" {
+ args = append(args, "--out", out)
+ }
+ return args
+ }
+ // apply_plan (and phase_out, with out): the dry run, then the apply.
+ tryPlan := func(p claimactivation.Plan, out string) (activationPhaseResult, error) {
t.Helper()
- if _, e := os.Lstat(path); !os.IsNotExist(e) {
- t.Fatal("present", path, e)
+ raw, _ := json.Marshal(p)
+ write(sd+"/plan.json", raw, 0600)
+ before := controlSnapshot()
+ if stdout, stderr, e := try(sd+"/carlos", append(runArgs(out), "--dry-run")...); e != nil {
+ return activationPhaseResult{}, fmt.Errorf("%s dry run: %v: %s %s", p.Phase, e, stdout, stderr)
+ }
+ if !reflect.DeepEqual(before, controlSnapshot()) {
+ t.Fatal("dry run wrote", p.Phase)
+ }
+ if out != "" {
+ absent(out)
+ }
+ stdout, stderr, e := try(sd+"/carlos", runArgs(out)...)
+ if e != nil {
+ return activationPhaseResult{}, fmt.Errorf("%s: %v: %s %s", p.Phase, e, stdout, stderr)
}
+ var r activationPhaseResult
+ check(json.Unmarshal([]byte(stdout), &r))
+ t.Log("phase", p.Phase, "revision", r.Revision, "writes", len(r.Writes))
+ return r, nil
+ }
+ applyPlan := func(p claimactivation.Plan, out string) activationPhaseResult {
+ t.Helper()
+ r, e := tryPlan(p, out)
+ check(e)
+ return r
+ }
+ phase := func(name string, p claimactivation.Plan) activationPhaseResult {
+ t.Helper()
+ p.Phase, p.Bucket, p.ExpectedRevision = name, bucket, refresh().Roster.Revision
+ return applyPlan(p, "")
}
- write("/opt/carlos/carlos", activationCarlosImage, 0755)
- write("/opt/carlos/carlos.prev", activationCarlosImage, 0755)
- write(d+"/release.key", []byte(base64.StdEncoding.EncodeToString(key.Seed())+"\n"), 0600)
- // The console release, as carlos ship and carlos promote leave it with
- // pointer digests on: a signed pointer naming the build and its manifest
- // digest.
+ // Step 5.1: roster and enrollment.
+ applyPlan(claimactivation.Plan{Phase: "initialize", Bucket: bucket}, "")
+ phase("configure", claimactivation.Plan{Regions: map[string]hostactors.Region{"ie": {Account: "640012953448", AWSRegion: "eu-west-1", Namespaces: []string{"platform"}, Pools: []string{"cohort1", "console"}}}})
+ boxInput := claimactivation.BoxInput{Reference: box, Namespace: "platform", Label: box, Region: "ie", Pool: "cohort1", AWSRegion: "eu-west-1", InstanceID: instance, Operator: "root-runner"}
+ phase("invite-box", claimactivation.Plan{Box: &boxInput})
+ phase("enroll-box", claimactivation.Plan{Box: &boxInput})
+ enrolled, ok := refresh().Roster.Boxes[box]
+ if !ok || enrolled.Pool != "cohort1" || enrolled.Cloud.InstanceID != instance || enrolled.State != "active" {
+ t.Fatal("enrolled box", enrolled)
+ }
+ token := make([]byte, 16)
+ _, err = rand.Read(token)
+ check(err)
+ consoleIn := hostactors.Console{InstanceID: consoleID, Enrollment: hex.EncodeToString(token), DeploymentBucket: bucket, Namespaces: []string{"platform"}, Compute: &hostactors.ComputeBinding{Box: box, Enrollment: enrolled.Enrollment}}
+ phase("invite-console", claimactivation.Plan{Console: &consoleIn})
+ phase("enroll-console", claimactivation.Plan{Console: &consoleIn})
+ if o := refresh(); o.Roster.Consoles[consoleID].Compute == nil || len(o.Roster.Boxes) != 1 {
+ t.Fatal("inspect", o.Roster)
+ }
+ // The box's staged inputs (/root/claim-trial on the trial's boxes).
+ bd := filepath.Join(d, "box")
+ check(os.MkdirAll(bd, 0700))
+ enrollmentRaw, _ := json.Marshal(hostfloor.Enrollment{Box: box, Enrollment: enrolled.Enrollment, InstanceID: enrolled.Cloud.InstanceID})
+ write(bd+"/"+box+"-enrollment.json", enrollmentRaw, 0600)
+
+ // Step 6.1: the armed policy, through the real binary's sign-policy.
+ floor := hostfloor.Floor{Min: 1, MinRelease: "v1.0.0"}
+ floorRaw, _ := json.Marshal(floor)
+ signed := applyPlan(claimactivation.Plan{Phase: "sign-policy", Bucket: bucket, ExpectedRevision: refresh().Roster.Revision, Floor: &floor}, sd+"/policy.json")
+ policyRaw, err := os.ReadFile(sd + "/policy.json")
+ check(err)
+ if len(signed.Writes) != 0 || signed.FloorSHA256 != hostfloor.Digest(floorRaw) || signed.OutputSHA256 != hostfloor.Digest(policyRaw) {
+ t.Fatal("sign-policy", signed.Result)
+ }
+ signerCLI(false, runArgs(sd+"/policy.json")...) // --out is never replaced
+ var policy hostfloor.Policy
+ check(json.Unmarshal(policyRaw, &policy))
+ if st, e := os.Stat(sd + "/policy.json"); e != nil || st.Mode().Perm() != 0600 || !bytes.Equal(policy.Floor, floorRaw) || policy.Mode != "armed" {
+ t.Fatal("policy file", st, e)
+ }
+ // The identity. The console release, as carlos ship and carlos promote
+ // leave it with pointer digests on (step 5.0a).
check(release.WriteDigestMarker(ctx, store, signer, "bab", "root-runner"))
check(release.Ship(ctx, store, "bab", release.Manifest{App: "console", Version: "abc1234", Kind: "binary", Artifacts: map[string]release.Artifact{"linux-arm64": {}}}, map[string][]byte{"linux-arm64": activationConsoleImage}))
check(signer.Promote(ctx, store, "bab", "console", "canary/console", "abc1234"))
-
- // The release identity, through the publish-identity phase over the
- // production S3 store (create-only PUT) against the loopback bucket. The
- // router hash comes from a signed checksum list; the runner's sums key is
- // this fixture's, because the vendor key cannot sign a fixture. The real
- // binary's own refusal of that list is checked below.
+ // publish-identity over the production S3 store (create-only PUT)
+ // against the loopback bucket. The router hash comes from a signed
+ // checksum list; the runner's sums key is this fixture's, because the
+ // vendor key cannot sign a fixture. The real binary's own refusal of that
+ // list is checked below.
s3, err := release.NewS3StoreIn(ctx, bucket, "eu-west-1")
check(err)
sumsPub, sumsKey, _ := ed25519.GenerateKey(nil)
identityRunner := claimactivation.Runner{Directory: hostactors.Directory{Store: s3, PublicKey: pub, Bucket: bucket}, Signer: signer, SumsKey: base64.StdEncoding.EncodeToString(sumsPub)}
sums := hostfloor.Digest(activationCarlosImage) + " carlos-linux-arm64\n" + hostfloor.Digest([]byte("amd64 deb")) + " carlos_1.0.0_amd64.deb\n" + hostfloor.Digest([]byte("arm64 deb")) + " carlos_1.0.0_arm64.deb\n"
- o, err = directory.Read(ctx)
- check(err)
- identityPlan := claimactivation.Plan{Phase: "publish-identity", Bucket: bucket, ExpectedRevision: o.Roster.Revision, Identity: &claimactivation.IdentityInput{ReleaseTag: "v1.0.0", MovesProtocol: 1, Sums: sums,
+ identityPlan := claimactivation.Plan{Phase: "publish-identity", Bucket: bucket, ExpectedRevision: refresh().Roster.Revision, Identity: &claimactivation.IdentityInput{ReleaseTag: "v1.0.0", MovesProtocol: 1, Sums: sums,
SumsSignature: base64.StdEncoding.EncodeToString(ed25519.Sign(sumsKey, []byte(sums))),
Console: claimactivation.ConsoleRelease{Account: "bab", App: "console", Channel: "canary/console", Version: "abc1234"}}}
preview, err := identityRunner.Run(ctx, identityPlan, true)
@@ -216,82 +484,89 @@ func TestRootActivationConsoleRuntimePayload(t *testing.T) {
}
published, err := identityRunner.Run(ctx, identityPlan, false)
check(err)
- identityOut, err := reserveActivationOutput(d+"/identity.json", false)
+ identityOut, err := reserveActivationOutput(sd+"/identity.json", false)
check(err)
check(identityOut.finish(published.Output))
identityOut.close()
+ identityRaw, err := os.ReadFile(sd + "/identity.json")
+ check(err)
planRaw, _ := json.Marshal(identityPlan)
- write(d+"/identity-plan.json", planRaw, 0600)
- carlos(false, "system", "claims", "run", "--plan", d+"/identity-plan.json", "--signing-key", d+"/release.key", "--public-key", pubArg, "--aws-region", "eu-west-1", "--out", d+"/identity-vendor.json")
- absent(d + "/identity-vendor.json")
+ write(sd+"/plan.json", planRaw, 0600)
+ if _, stderr := signerCLI(false, runArgs(sd+"/identity-vendor.json")...); !strings.Contains(stderr, sd+"/identity-vendor.json was kept. run it again with a new --out path") {
+ t.Fatal("vendor-key refusal", stderr)
+ }
+ if st, e := os.Stat(sd + "/identity-vendor.json"); e != nil || st.Size() != 0 {
+ t.Fatal("the refused run's reservation", st, e)
+ }
+ // Staged on the box, byte for byte (box_put).
+ write(bd+"/policy.json", policyRaw, 0600)
+ write(bd+"/identity.json", identityRaw, 0600)
- // The armed policy, through the real binary's sign-policy phase, to --out.
- floor := hostfloor.Floor{Min: 1, MinRelease: "v1.0.0"}
- floorRaw, _ := json.Marshal(floor)
- o, err = directory.Read(ctx)
- check(err)
- planRaw, _ = json.Marshal(claimactivation.Plan{Phase: "sign-policy", Bucket: bucket, ExpectedRevision: o.Roster.Revision, Floor: &floor})
- write(d+"/sign-policy.json", planRaw, 0600)
- sign := []string{"system", "claims", "run", "--plan", d + "/sign-policy.json", "--signing-key", d + "/release.key", "--public-key", pubArg, "--aws-region", "eu-west-1", "--out", d + "/policy.json"}
- before := activationStoreSnapshot(t, store)
- carlos(true, append(append([]string{}, sign...), "--dry-run")...)
- absent(d + "/policy.json")
- t.Log("sign-policy", carlos(true, sign...))
- carlos(false, sign...) // --out is never replaced
- if !reflect.DeepEqual(before, activationStoreSnapshot(t, store)) {
- t.Fatal("sign-policy wrote to the bucket")
- }
- policyRaw, err := os.ReadFile(d + "/policy.json")
- check(err)
- var policy hostfloor.Policy
- check(json.Unmarshal(policyRaw, &policy))
- if st, e := os.Stat(d + "/policy.json"); e != nil || st.Mode().Perm() != 0600 || !bytes.Equal(policy.Floor, floorRaw) || policy.Mode != "armed" {
- t.Fatal("policy file", st, e)
+ // Step 6.2 as the trial ran it: guard-arm refuses the updated box, whose
+ // carlos.prev is the release before, and writes nothing.
+ armLeo := []string{"system", "guard-arm", "--policy", bd + "/policy.json", "--identity", bd + "/identity.json", "--enrollment", bd + "/" + box + "-enrollment.json"}
+ _, guardBefore := os.Lstat(systemGuardDir(opt))
+ if _, stderr := carlos(false, armLeo...); !strings.Contains(stderr, "hostfloor: authenticated floor required") {
+ t.Fatal("the trial's stop did not reproduce", stderr)
+ }
+ absent("/etc/carlos/host-actor.json")
+ absent("/etc/systemd/system/carlos-edge.service.d/90-host-floor.conf")
+ if _, e := os.Lstat(systemGuardDir(opt)); os.IsNotExist(e) != os.IsNotExist(guardBefore) {
+ t.Fatal("the refused guard-arm changed the guard directory", e)
+ }
+ t.Log("step 6.2 stop reproduced: guard-arm refuses carlos.prev v0.9.0")
+ // install-channel without --policy is the trial's loop: no floor yet.
+ if _, stderr := carlos(false, "system", "install-channel"); !strings.HasSuffix("\n"+strings.TrimSpace(stderr), "\nno host floor is published, so this box could not arm its guard and nothing was installed") {
+ t.Fatal("install-channel without --policy", stderr)
+ }
+ // R1: install-channel --policy on the box.
+ if out, _ := carlos(true, "system", "install-channel", "--policy", bd+"/policy.json"); strings.TrimSpace(out) != "replaced the rollback copy carlos.prev with v1.0.0 (v1.0.0) from the stable channel, the release carlos already runs. the release carlos.prev held before is no longer on this box" {
+ t.Fatal("install-channel --policy", out)
+ }
+ cur, _ = os.ReadFile(systemBinaryPath(opt))
+ prev, _ = os.ReadFile(systemPrevPath(opt))
+ if !bytes.Equal(cur, activationCarlosImage) || !bytes.Equal(prev, activationCarlosImage) {
+ t.Fatal("install-channel --policy did not leave v1.0.0 in both slots")
+ }
+ if held, e := os.ReadFile(channelIdentityPath(opt)); e != nil || !bytes.Equal(held, identityRaw) {
+ t.Fatal("the channel identity is not the published one", e)
+ }
+ if out, _ := carlos(true, "system", "install-channel", "--policy", bd+"/policy.json"); strings.TrimSpace(out) != "v1.0.0 (v1.0.0) from the stable channel is already installed" {
+ t.Fatal("install-channel --policy again", out)
+ }
+ // Readers (P3's box line): both slots are the trial release.
+ if c, p := readers(); c != "carlos v1.0.0" || p != "carlos v1.0.0" {
+ t.Fatal("readers after install-channel", c, p)
+ }
+ // Step 6.2: arm the hosting box (leo's form).
+ armed := time.Now()
+ carlos(true, armLeo...)
+ if raw, e := os.ReadFile("/etc/carlos/host-actor.json"); e != nil || !bytes.Equal(raw, enrollmentRaw) {
+ t.Fatal("host-actor.json", e)
+ }
+ if raw, e := os.ReadFile("/etc/systemd/system/carlos-edge.service.d/90-host-floor.conf"); e != nil || !strings.Contains(string(raw), "guard-exec edge") {
+ t.Fatal("edge drop-in", e)
}
+ t.Log("step 6.2 passed: guard-arm armed the updated box")
- // The protected console copies, from a carlos-owned versions tree as on
- // the dev console box, through the real binary's install-console.
+ // Step 6.3: the console box. The console release's build, staged in a
+ // carlos-owned versions tree as on the dev console box.
+ uid, gid := activationServiceIDs(t)
+ check(os.Chmod(d, 0755))
source := d + "/versions/abc1234/console"
write(source, activationConsoleImage, 0755)
write(d+"/versions/other/console", append([]byte("not the release "), activationConsoleImage[:64]...), 0755)
for _, p := range []string{d + "/versions", d + "/versions/abc1234", source} {
check(os.Chown(p, uid, gid))
}
- install := func(src string) []string {
- return []string{"system", "claims", "install-console", "--identity", d + "/identity.json", "--source", src, "--bucket", bucket, "--aws-region", "eu-west-1", "--public-key", pubArg}
- }
- carlos(false, install(d+"/versions/other/console")...)
- carlos(false, install("/opt/carlos/carlos")...)
- absent("/opt/carlos/system/console")
- carlos(true, append(install(source), "--dry-run")...)
- absent("/opt/carlos/system/console")
- t.Log("install-console", carlos(true, install(source)...))
- for _, p := range []string{"/opt/carlos/system/console/console", "/opt/carlos/system/console/console.prev"} {
- body, e := os.ReadFile(p)
- st, se := os.Stat(p)
- if e != nil || se != nil || !bytes.Equal(body, activationConsoleImage) || st.Mode().Perm() != 0755 || st.Sys().(*syscall.Stat_t).Uid != 0 || st.Sys().(*syscall.Stat_t).Gid != 0 {
- t.Fatal("protected console", p, e, se)
- }
- }
- if out := carlos(true, install(source)...); !strings.Contains(out, `"already_installed": true`) {
- t.Fatal("rerun was not a no-op", out)
- }
- local := hostfloor.Enrollment{Box: "sophie", Enrollment: c.Compute.Enrollment, InstanceID: instance}
- localRaw, _ := json.Marshal(local)
- write(d+"/enrollment.json", localRaw, 0600)
- check(os.MkdirAll(d+"/data", 0755))
- check(os.Chown(d+"/data", uid, gid))
- check(os.Chmod(d, 0755))
- write("/etc/carlos/host.env", []byte(strings.Join(environment[1:], "\n")+"\n"), 0600)
- carlos(true, "system", "guard-arm", "--policy", d+"/policy.json", "--identity", d+"/identity.json", "--enrollment", d+"/enrollment.json", "--console", "/opt/carlos/system/console/console")
- dbPath := d + "/data/console.db"
+ // The box-only console.service and its database.
+ dbPath := "/data/console.db"
st, err := console.Open(dbPath)
check(err)
now := time.Now().UTC()
check(st.CreateAccount(123, "bab", "fixture", "operator@example.test", now))
_, err = st.CreateAppIn(123, "trial", "operator@example.test", now)
check(err)
- // OpenGated's schema migration is also used on the fixture before serving.
check(st.Close())
st, err = console.OpenGated(dbPath)
check(err)
@@ -307,8 +582,8 @@ func TestRootActivationConsoleRuntimePayload(t *testing.T) {
check(e)
return token
}
- token := tokenFor("operator@example.test")
- other := tokenFor("member@example.test")
+ operatorToken := tokenFor("operator@example.test")
+ memberToken := tokenFor("member@example.test")
check(st.SetOperator("operator@example.test", true))
check(st.Close())
for _, p := range []string{dbPath, dbPath + "-wal", dbPath + "-shm"} {
@@ -320,103 +595,160 @@ func TestRootActivationConsoleRuntimePayload(t *testing.T) {
check(err)
addr := ln.Addr().String()
check(ln.Close())
- unit := "[Service]\nType=notify\nUser=carlos\nGroup=carlos\nEnvironmentFile=/etc/carlos/host.env\nEnvironment=CONSOLE_ADDR=" + addr + " CONSOLE_ORIGIN=http://" + addr + " CONSOLE_KEYMAIL=http://127.0.0.1:1 CONSOLE_DB=" + dbPath + "\nExecStart=/opt/carlos/system/console/console serve\nRestart=no\nTimeoutStartSec=30s\nNoNewPrivileges=yes\nProtectSystem=strict\nReadWritePaths=" + d + "/data\nPrivateTmp=yes\nIPAddressDeny=any\nIPAddressAllow=127.0.0.1 169.254.169.254\n"
+ write("/etc/carlos/console-fixture.env", []byte("CARLOS_RELEASE_KEY="+base64.StdEncoding.EncodeToString(key.Seed())+"\n"), 0600)
+ unit := "[Service]\nType=notify\nUser=carlos\nGroup=carlos\nEnvironmentFile=/etc/carlos/host.env\nEnvironmentFile=/etc/carlos/console-fixture.env\nEnvironment=CONSOLE_ADDR=" + addr + " CONSOLE_ORIGIN=http://" + addr + " CONSOLE_KEYMAIL=http://127.0.0.1:1 CONSOLE_DB=" + dbPath + "\nExecStart=/opt/carlos/system/console/console serve\nRestart=no\nTimeoutStartSec=30s\nNoNewPrivileges=yes\nProtectSystem=strict\nReadWritePaths=/data\nPrivateTmp=yes\nIPAddressDeny=any\nIPAddressAllow=127.0.0.1 169.254.169.254\n"
write("/etc/systemd/system/console.service", []byte(unit), 0644)
- config := func(enabled bool) {
- t.Helper()
- o, e := directory.Read(ctx)
- check(e)
- wire, _, e := store.Get(ctx, hostactors.Key)
+ ctl("daemon-reload")
+ // P9: no console mail name anywhere the console's environment comes from.
+ for _, p := range []string{"/etc/carlos/host.env", "/etc/carlos/console-fixture.env", "/etc/systemd/system/console.service"} {
+ raw, e := os.ReadFile(p)
check(e)
- write(d+"/roster.json", wire, 0600)
- args := []string{"console-config", "--roster", d + "/roster.json", "--expect-revision", fmt.Sprint(o.Roster.Revision), "--bucket", bucket, "--console-id", c.InstanceID, "--public-key", base64.StdEncoding.EncodeToString(pub)}
- if enabled {
- args = append(args, "--enable-console-runtime")
+ for _, name := range []string{"CARLOS_CONSOLE_MAIL_FROM", "CARLOS_MAIL_REGION", "CARLOS_MAIL_PLATFORM_SUFFIX", "CARLOS_MAIL_DMARC_RUA", "CARLOS_MAIL_SNS_TOPIC_ARN", "CARLOS_MAIL_OPERATORS", "CARLOS_MAIL_LEGACY_SWEEP"} {
+ if bytes.Contains(raw, []byte(name)) {
+ t.Fatal("P9: SET", name, p)
+ }
}
- var output bytes.Buffer
- check(runSystemClaims(append(append([]string{}, args...), "--dry-run"), &output))
- check(runSystemClaims(args, &output))
}
- config(false)
- ctl := func(args ...string) {
+ // box_inspect: the box reads the roster with its own credentials.
+ boxInspect := func() uint64 {
t.Helper()
- out, e := exec.CommandContext(ctx, "systemctl", args...).CombinedOutput()
- if e != nil {
- t.Fatal(args, string(out), e)
+ o := inspectWith(systemBinaryPath(opt))
+ write(bd+"/roster.json", o.SignedRoster, 0600)
+ return o.Roster.Revision
+ }
+ rev := boxInspect()
+ install := func(src string) []string {
+ return []string{"system", "claims", "install-console", "--identity", bd + "/identity.json", "--source", src, "--bucket", bucket, "--aws-region", "eu-west-1", "--public-key", pubArg}
+ }
+ carlos(false, install(d+"/versions/other/console")...)
+ carlos(false, install(systemBinaryPath(opt))...)
+ absent("/opt/carlos/system/console")
+ carlos(true, append(install(source), "--dry-run")...)
+ absent("/opt/carlos/system/console")
+ out, _ = carlos(true, install(source)...)
+ t.Log("install-console", out)
+ for _, p := range []string{"/opt/carlos/system/console/console", "/opt/carlos/system/console/console.prev"} {
+ body, e := os.ReadFile(p)
+ st, se := os.Stat(p)
+ if e != nil || se != nil || !bytes.Equal(body, activationConsoleImage) || st.Mode().Perm() != 0755 || st.Sys().(*syscall.Stat_t).Uid != 0 || st.Sys().(*syscall.Stat_t).Gid != 0 {
+ t.Fatal("protected console", p, e, se)
+ }
+ }
+ if out, _ := carlos(true, install(source)...); !strings.Contains(out, `"already_installed": true`) {
+ t.Fatal("rerun was not a no-op", out)
+ }
+ // guard-arm on the console box (sophie's form).
+ carlos(true, append(append([]string{}, armLeo...), "--console", "/opt/carlos/system/console/console")...)
+ dropIn, err := os.ReadFile("/etc/systemd/system/console.service.d/90-host-floor.conf")
+ check(err)
+ if string(dropIn) != hostfloor.ConsoleGuardDropIn("/opt/carlos/system/console/console") {
+ t.Fatal("guard drop-in", string(dropIn))
+ }
+ consoleConfig := func(enable bool) {
+ t.Helper()
+ args := []string{"system", "claims", "console-config", "--roster", bd + "/roster.json", "--expect-revision", fmt.Sprint(rev), "--bucket", bucket, "--console-id", consoleID, "--public-key", pubArg}
+ if enable {
+ args = append(args, "--enable-console-runtime")
}
+ carlos(true, append(append([]string{}, args...), "--dry-run")...)
+ carlos(true, args...)
+ }
+ consoleConfig(false)
+ ctl("restart", "console.service")
+ if state := ctl("is-active", "console.service"); state != "active" {
+ t.Fatal("console", state)
}
- ctl("start", "console.service")
- report := func(enabled bool, previous string) hostactors.Report {
+ consoleReported := func(enabled bool, previous string) hostactors.Report {
t.Helper()
- deadline := time.Now().Add(20 * time.Second)
+ deadline := time.Now().Add(30 * time.Second)
for time.Now().Before(deadline) {
- raw, _, e := store.Get(ctx, "status/consoles/dev-console.json")
+ raw, _, e := store.Get(ctx, "status/consoles/"+consoleID+".json")
var r hostactors.Report
- if e == nil && json.Unmarshal(raw, &r) == nil && r.HostClaimEnabled == enabled && r.ProcessID != previous {
+ if e == nil && json.Unmarshal(raw, &r) == nil && r.HostClaimEnabled == enabled && r.ProcessID != "" && r.ProcessID != previous {
return r
}
- time.Sleep(100 * time.Millisecond)
+ time.Sleep(200 * time.Millisecond)
}
- t.Fatal("console did not report", enabled)
+ t.Fatal("console did not report", enabled, journal("console.service", 60))
return hostactors.Report{}
}
- first := report(false, "")
- // The console runs under the guard drop-in as the service user, from
- // the protected copy's bytes.
- pidOut, err := exec.Command("systemctl", "show", "-p", "MainPID", "--value", "console.service").Output()
- check(err)
- pid := strings.TrimSpace(string(pidOut))
- statusRaw, err := os.ReadFile("/proc/" + pid + "/status")
+ first := consoleReported(false, "")
+ // The console runs under the guard drop-in as the service user, from the
+ // protected copy's bytes.
+ pidOut := ctl("show", "-p", "MainPID", "--value", "console.service")
+ statusRaw, err := os.ReadFile("/proc/" + pidOut + "/status")
check(err)
if !strings.Contains(string(statusRaw), fmt.Sprintf("\nUid:\t%d\t%d\t", uid, uid)) {
t.Fatal("console is not the service user", string(statusRaw))
}
- exe, err := os.ReadFile("/proc/" + pid + "/exe")
+ exe, err := os.ReadFile("/proc/" + pidOut + "/exe")
check(err)
if hostfloor.Digest(exe) != hostfloor.Digest(activationConsoleImage) {
t.Fatal("console process image differs from the protected copy")
}
- dropIn, err := os.ReadFile("/etc/systemd/system/console.service.d/90-host-floor.conf")
- check(err)
- if string(dropIn) != hostfloor.ConsoleGuardDropIn("/opt/carlos/system/console/console") {
- t.Fatal("guard drop-in", string(dropIn))
+ // The agent's own box report, once it has seen the armed guard: the
+ // evidence the console report and the floor publication read. Nothing
+ // here writes it.
+ boxReported := func() {
+ t.Helper()
+ deadline := time.Now().Add(9 * time.Minute)
+ for {
+ raw, _, e := store.Get(ctx, "status/boxes/"+box+".json")
+ var b status.Box
+ if e == nil && json.Unmarshal(raw, &b) == nil && b.Metrics != nil && b.Metrics.FloorGuardReady && b.Metrics.FloorSHA256 == hostfloor.Digest(floorRaw) && b.Metrics.Enrollment == enrolled.Enrollment && b.GeneratedAt.After(armed) {
+ t.Log("agent box report", b.GeneratedAt, b.Metrics.Release, b.Metrics.BinarySHA256)
+ return
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("the agent did not report the armed guard: %s\n%s", raw, journal("carlos-edge.service", 80))
+ }
+ time.Sleep(5 * time.Second)
+ }
}
- // Box reports are fixture telemetry. The console process entry is produced
- // by the real root observer below, never copied from the console report.
- put := func(k string, v any) {
+ boxReported()
+ // console-report: dry run, then the report. The agent's next box report
+ // replaces the console process entry, so a report that loses that race
+ // is run again (the runbook's rule).
+ consoleReport := func() {
t.Helper()
- raw, _ := json.Marshal(v)
- _, tag, _ := store.Get(ctx, k)
- _, e := store.Put(ctx, k, raw, tag)
- check(e)
+ var last error
+ for attempt := 1; attempt <= 3; attempt++ {
+ rev = boxInspect()
+ args := []string{"system", "claims", "console-report", "--bucket", bucket, "--aws-region", "eu-west-1", "--expect-revision", fmt.Sprint(rev), "--console-id", consoleID, "--public-key", pubArg}
+ if stdout, stderr, e := try(systemBinaryPath(opt), append(append([]string{}, args...), "--dry-run")...); e != nil {
+ last = fmt.Errorf("dry run: %v: %s %s", e, stdout, stderr)
+ } else if stdout, stderr, e := try(systemBinaryPath(opt), args...); e != nil {
+ last = fmt.Errorf("%v: %s %s", e, stdout, stderr)
+ } else {
+ t.Log("console-report", stdout)
+ return
+ }
+ t.Log("console-report attempt", attempt, last)
+ time.Sleep(3 * time.Second)
+ }
+ t.Fatal("console-report", last)
}
- for ref, b := range next.Boxes {
- put("status/boxes/"+ref+".json", status.Box{GeneratedAt: time.Now().UTC(), Metrics: &status.BoxMetrics{Enrollment: b.Enrollment, InstanceID: b.Cloud.InstanceID, Release: "v1.0.0", ReleaseTag: "v1.0.0", Platform: "linux-arm64", BinarySHA256: hostfloor.Digest(activationCarlosImage), MovesProtocol: 1, BootID: first.BootID, FloorSHA256: hostfloor.Digest(floorRaw), FloorGuardReady: true}})
+ consoleReport()
+ t.Log("step 6 passed")
+
+ // Step 7.1: readers. Every slot is the trial release.
+ if c, p := readers(); c != "carlos v1.0.0" || p != "carlos v1.0.0" {
+ t.Fatal("readers", c, p)
}
- o, err = directory.Read(ctx)
- check(err)
- if _, err = runner.Run(ctx, claimactivation.Plan{Phase: "publish-floor", Bucket: bucket, ExpectedRevision: o.Roster.Revision, Floor: &floor}, true); err == nil {
- t.Fatal("published without independent process observation")
+ // Step 7.2: floor and home, immediately after the console's report.
+ floorPlan := func() claimactivation.Plan {
+ return claimactivation.Plan{Phase: "publish-floor", Bucket: bucket, ExpectedRevision: refresh().Roster.Revision, Floor: &floor}
}
- publish := func() {
- t.Helper()
- o, e := directory.Read(ctx)
- check(e)
- before := activationStoreSnapshot(t, store)
- p, e := publishActivationConsoleReport(ctx, store, pub, bucket, c.InstanceID, o.Roster.Revision, true)
- check(e)
- if !reflect.DeepEqual(before, activationStoreSnapshot(t, store)) {
- t.Fatal("report dry run wrote")
- }
- cmd := exec.CommandContext(ctx, "/opt/carlos/carlos", "system", "claims", "console-report", "--bucket", bucket, "--aws-region", "eu-west-1", "--expect-revision", fmt.Sprint(o.Roster.Revision), "--console-id", c.InstanceID, "--public-key", base64.StdEncoding.EncodeToString(pub))
- cmd.Env = environment
- if output, e := cmd.CombinedOutput(); e != nil {
- t.Fatal("report CLI", string(output), e)
+ for attempt := 1; ; attempt++ {
+ if _, e := tryPlan(floorPlan(), ""); e == nil {
+ break
+ } else if attempt == 3 {
+ t.Fatal("publish-floor", e)
+ } else {
+ t.Log("publish-floor attempt", attempt, e)
}
- t.Log("authenticated process", p.ProcessID, p.BinarySHA256)
+ consoleReport()
}
- publish()
- phase(claimactivation.Plan{Phase: "publish-floor", Floor: &floor})
// The floor publish-floor wrote is the armed policy's, byte for byte.
for k, want := range map[string][]byte{hostfloor.Key: policy.Floor, hostfloor.AuthKey: policy.Authorization} {
got, _, e := store.Get(ctx, k)
@@ -424,38 +756,99 @@ func TestRootActivationConsoleRuntimePayload(t *testing.T) {
t.Fatal("published floor differs from the armed policy", k, e)
}
}
- phase(claimactivation.Plan{Phase: "home", HomeRegion: "ie"})
- binding := hostactors.CensusKeyBinding{Bucket: bucket, Box: "leo", Enrollment: next.Boxes["leo"].Enrollment}
- proof, err := installActivationEvidenceKey(d+"/leo-evidence.key", binding, false)
- check(err)
- phase(claimactivation.Plan{Phase: "bind-key", Binding: &proof.Binding, Proof: proof.Proof})
- config(true)
+ phase("home", claimactivation.Plan{HomeRegion: "ie"})
+ // Step 7.3: the box's evidence key, then bind-key on the signer.
+ rev = boxInspect()
+ evidence := []string{"system", "claims", "evidence-key", "--roster", bd + "/roster.json", "--expect-revision", fmt.Sprint(rev), "--bucket", bucket, "--box", box, "--enrollment", enrolled.Enrollment, "--public-key", pubArg}
+ carlos(true, append(append([]string{}, evidence...), "--dry-run")...)
+ absent("/etc/carlos/host-evidence.key")
+ proofOut, _ := carlos(true, evidence...)
+ write(bd+"/key-proof.json", []byte(proofOut), 0600)
+ var keyProof struct {
+ Proof activationKeyProof `json:"key_proof"`
+ }
+ check(json.Unmarshal([]byte(proofOut), &keyProof))
+ if fi, e := os.Lstat("/etc/carlos/host-evidence.key"); e != nil || fi.Mode().Perm() != 0600 || fi.Sys().(*syscall.Stat_t).Uid != 0 || fi.Sys().(*syscall.Stat_t).Nlink != 1 {
+ t.Fatal("host-evidence.key", fi, e)
+ }
+ phase("bind-key", claimactivation.Plan{Binding: &keyProof.Proof.Binding, Proof: keyProof.Proof.Proof})
+ if o := refresh(); o.Roster.HomeRegion != "ie" || o.Roster.Publication != "published" || o.Roster.RequestLimitNS != 120000000000 {
+ t.Fatal("roster after step 7.3", o.Roster.HomeRegion, o.Roster.Publication, o.Roster.RequestLimitNS)
+ }
+ // Step 7.4: the console runtime.
+ rev = boxInspect()
+ consoleConfig(true)
ctl("restart", "console.service")
- second := report(true, first.ProcessID)
- publish()
+ if state := ctl("is-active", "console.service"); state != "active" {
+ t.Fatal("console", state)
+ }
+ second := consoleReported(true, first.ProcessID)
+ consoleReport()
+ t.Log("step 7 passed")
+
+ // The console runtime's own refusals, as before.
raw, err := os.ReadFile("/etc/carlos/host.env")
check(err)
if bytes.Contains(raw, []byte("CARLOS_HOST_CLAIM=")) {
t.Fatal("enabled host execution")
}
- o, err = directory.Read(ctx)
+ o, err := (hostactors.Directory{Store: store, PublicKey: pub, Bucket: bucket}).Read(ctx)
check(err)
forged := second
forged.ProcessID = first.ProcessID
- if _, err = activationConsoleProcess(ctx, pub, bucket, o, c.InstanceID, forged); err == nil {
+ if _, err = activationConsoleProcess(ctx, pub, bucket, o, consoleID, forged); err == nil {
t.Fatal("accepted old process")
}
forged = second
forged.BinarySHA256 = strings.Repeat("a", 64)
- if _, err = activationConsoleProcess(ctx, pub, bucket, o, c.InstanceID, forged); err == nil {
+ if _, err = activationConsoleProcess(ctx, pub, bucket, o, consoleID, forged); err == nil {
t.Fatal("accepted false process digest")
}
- activationResetFixture(t, store, "http://"+addr, token, other)
+ activationResetFixture(t, store, "http://"+addr, operatorToken, memberToken, box, enrolled.Enrollment)
+ // Observation for the report: the agent's next box report is written
+ // whole (status.WriteBox), so whether the console process entry
+ // console-report added survives it is recorded here, not asserted.
+ reportedAt := time.Now()
+ for deadline := time.Now().Add(7 * time.Minute); time.Now().Before(deadline); time.Sleep(5 * time.Second) {
+ raw, _, e := store.Get(ctx, "status/boxes/"+box+".json")
+ var b status.Box
+ if e == nil && json.Unmarshal(raw, &b) == nil && b.Metrics != nil && b.GeneratedAt.After(reportedAt) {
+ t.Log("OBSERVATION: the agent's next box report at", b.GeneratedAt, "carries", len(b.Metrics.ConsoleProcesses), "console process entries")
+ break
+ }
+ }
ctl("stop", "console.service")
- if _, err = publishActivationConsoleReport(ctx, store, pub, bucket, c.InstanceID, o.Roster.Revision, false); err == nil {
+ if _, err = publishActivationConsoleReport(ctx, store, pub, bucket, consoleID, o.Roster.Revision, false); err == nil {
t.Fatal("reported stopped console")
}
- t.Log("PASS real guard-arm, guarded console, claims-off/on reports, floor/home/key sequence, authenticated reset and refusals")
+ t.Log("PASS steps 6 and 7 on a box made by bootstrap and the real update: the 6.2 stop reproduced, install-channel --policy, guard-arm in both forms, the guarded console, floor, home, key and console runtime, with the agent's own box reports")
+}
+
+// activationServiceIDs is the service user bootstrap created.
+func activationServiceIDs(t *testing.T) (int, int) {
+ t.Helper()
+ u, err := user.Lookup(serviceUser)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var uid, gid int
+ if _, err := fmt.Sscan(u.Uid, &uid); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := fmt.Sscan(u.Gid, &gid); err != nil {
+ t.Fatal(err)
+ }
+ return uid, gid
+}
+
+// activationDirNames is the set of names in dir; a missing dir is empty.
+func activationDirNames(dir string) map[string]bool {
+ names := map[string]bool{}
+ ents, _ := os.ReadDir(dir)
+ for _, e := range ents {
+ names[e.Name()] = true
+ }
+ return names
}
func activationStoreSnapshot(t *testing.T, s release.Store) map[string]string {
@@ -467,6 +860,9 @@ func activationStoreSnapshot(t *testing.T, s release.Store) map[string]string {
result := map[string]string{}
for _, k := range keys {
raw, tag, e := s.Get(t.Context(), k)
+ if errors.Is(e, os.ErrNotExist) {
+ continue // removed between the list and the read
+ }
if e != nil {
t.Fatal(e)
}
@@ -569,7 +965,10 @@ func activationS3Fixture(t *testing.T, s release.Store) string {
return server.URL
}
-func activationResetFixture(t *testing.T, s release.Store, origin, token, other string) {
+// activationResetFixture is the authenticated reset of an exhausted wake
+// retry through the guarded console. Its claim state is fixture state; box
+// and enrollment are the enrolled box's.
+func activationResetFixture(t *testing.T, s release.Store, origin, token, other, box, enrollment string) {
t.Helper()
ctx := t.Context()
check := func(e error) {
@@ -585,10 +984,10 @@ func activationResetFixture(t *testing.T, s release.Store, origin, token, other
check(err)
o, err = hostclaim.Update(ctx, s, o, hostclaim.BeginCreate(hostclaim.NewHolder{Family: "instance", Key: "apps/bab/trial/instances/" + host + ".json", Account: "bab", App: "trial", Host: host, Region: "ie", RecordRegion: "ie", CreatedAt: now}, record, "create", "dev-console", now))
check(err)
- o, err = hostclaim.Update(ctx, s, o, hostclaim.CompleteCreate("create", "leo"))
+ o, err = hostclaim.Update(ctx, s, o, hostclaim.CompleteCreate("create", box))
check(err)
for i := 0; i < hostclaim.WakeRetryLimit; i++ {
- a := hostclaim.WakeRetryAttempt{ID: fmt.Sprint(i), Box: "leo", Enrollment: strings.Repeat("2", 32), BootID: "fixture", ControllerID: "fixture", InvocationID: "fixture", LaunchInputs: "fixture"}
+ a := hostclaim.WakeRetryAttempt{ID: fmt.Sprint(i), Box: box, Enrollment: enrollment, BootID: "fixture", ControllerID: "fixture", InvocationID: "fixture", LaunchInputs: "fixture"}
o, err = hostclaim.Update(ctx, s, o, hostclaim.ReserveWakeRetryAttempt(hostclaim.WakeRetryReservation{HolderIncarnation: o.Claim.Holder.Incarnation, EpisodeID: "exhausted", Attempt: a, Now: now}))
check(err)
o, err = hostclaim.Update(ctx, s, o, hostclaim.FailWakeRetryAttempt(hostclaim.WakeRetryResult{HolderIncarnation: o.Claim.Holder.Incarnation, EpisodeID: "exhausted", Attempt: a, Now: now}))