| 1 | //go:build linux && rootrunner |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "encoding/json" |
| 7 | "github.com/carlosframework/platform/internal/registry" |
| 8 | "github.com/carlosframework/platform/internal/release" |
| 9 | "github.com/carlosframework/platform/internal/rollout" |
| 10 | "github.com/carlosframework/platform/internal/status" |
| 11 | "os" |
| 12 | "os/exec" |
| 13 | "path/filepath" |
| 14 | "syscall" |
| 15 | "testing" |
| 16 | "time" |
| 17 | ) |
| 18 | |
| 19 | func TestPlan14RootRetryProjectionDelivery(t *testing.T) { |
| 20 | if os.Getenv("PLAN14_RETRY_READER") == "1" { |
| 21 | report := rootRetryStatus(&status.RolloutReport{Hosts: map[string]status.HostRollout{"h.example.com": {HeldBook: "forged", HeldEpoch: 999}}}) |
| 22 | raw, _ := json.Marshal(report) |
| 23 | os.Stdout.Write(raw) |
| 24 | os.Exit(0) |
| 25 | } |
| 26 | if os.Geteuid() != 0 { |
| 27 | t.Fatal("root required") |
| 28 | } |
| 29 | if err := os.Mkdir(rootRetryProjection, 0755); err != nil { |
| 30 | t.Fatal(err) |
| 31 | } |
| 32 | defer os.RemoveAll(rootRetryProjection) |
| 33 | state := testRootFailureStore(t) |
| 34 | book, err := rollout.InitializeRootFailures(state, "box/en/1", nil) |
| 35 | if err != nil { |
| 36 | t.Fatal(err) |
| 37 | } |
| 38 | rt := registry.Route{Host: "h.example.com", Account: "bab", App: "km"} |
| 39 | if _, err = book.Record(rollout.FailureRecord{Account: rt.Account, App: rt.App, Host: rt.Host, Build: "v1", HeldDown: []string{rt.Host}}); err != nil { |
| 40 | t.Fatal(err) |
| 41 | } |
| 42 | p := rootUpdateSource{book: book, publish: publishRootRetry} |
| 43 | if err = p.publishRetry(rootUpdateRecord{Route: rt}, "v1"); err != nil { |
| 44 | t.Fatal(err) |
| 45 | } |
| 46 | exe, err := os.Executable() |
| 47 | if err != nil { |
| 48 | t.Fatal(err) |
| 49 | } |
| 50 | cmd := exec.Command(exe, "-test.run=^TestPlan14RootRetryProjectionDelivery$") |
| 51 | cmd.Env = append(os.Environ(), "PLAN14_RETRY_READER=1") |
| 52 | // /root is private on the runner: the child uses an executable descriptor |
| 53 | // opened before dropping credentials, without making the runner binary public. |
| 54 | binary, err := os.Open(exe) |
| 55 | if err != nil { |
| 56 | t.Fatal(err) |
| 57 | } |
| 58 | defer binary.Close() |
| 59 | cmd.Path = "/proc/self/fd/3" |
| 60 | cmd.Args[0] = cmd.Path |
| 61 | cmd.ExtraFiles = []*os.File{binary} |
| 62 | cmd.SysProcAttr = &syscall.SysProcAttr{Credential: &syscall.Credential{Uid: 65534, Gid: 65534}} |
| 63 | raw, err := cmd.Output() |
| 64 | if err != nil { |
| 65 | t.Fatal(err) |
| 66 | } |
| 67 | var report status.RolloutReport |
| 68 | if err = json.Unmarshal(raw, &report); err != nil { |
| 69 | t.Fatal(err, string(raw)) |
| 70 | } |
| 71 | h := report.Hosts[rt.Host] |
| 72 | if h.HeldBook != book.ID() || h.HeldEpoch != book.PublishedEpoch(rt.Account, rt.App, "v1", rt.Host) { |
| 73 | t.Fatal("agent reported forged retry", h) |
| 74 | } |
| 75 | legacy, err := rollout.OpenFailureBook(t.TempDir()) |
| 76 | if err != nil { |
| 77 | t.Fatal(err) |
| 78 | } |
| 79 | if _, err = legacy.Record(rollout.FailureRecord{Account: rt.Account, App: rt.App, Host: rt.Host, Build: "v1", HeldDown: []string{rt.Host}}); err != nil { |
| 80 | t.Fatal(err) |
| 81 | } |
| 82 | legacyEpoch := legacy.PublishedEpoch(rt.Account, rt.App, "v1", rt.Host) |
| 83 | rt.Kind, rt.Status, rt.Version, rt.Channel, rt.Hibernate = "instance", "ready", "v1", "canary/x", true |
| 84 | if !rollout.Restrictable(rt) { |
| 85 | t.Fatal("fixture route not restrictable") |
| 86 | } |
| 87 | hold := &failureHolds{book: legacy, get: func(string) (*registry.Route, error) { return &rt, nil }} |
| 88 | old := declinedHolds.Swap(hold) |
| 89 | defer declinedHolds.Store(old) |
| 90 | // A root retry has no effect on the separate local mirror until root's |
| 91 | // durable acknowledgement names the exact local snapshot it observed. |
| 92 | |
| 93 | seed, public, _ := release.GenerateKey() |
| 94 | priv, _ := release.ParseSigningKey(seed, "test") |
| 95 | pub, _ := release.ParsePublicKey(public, "test") |
| 96 | wire, err := release.SignHostRetry(release.RootSigner(priv), rt.Account, rt.App, rt.Host, h.HeldBy, h.HeldBook, h.HeldEpoch, time.Now()) |
| 97 | if err != nil { |
| 98 | t.Fatal(err) |
| 99 | } |
| 100 | data := t.TempDir() |
| 101 | t.Setenv("CARLOS_DATA_DIR", data) |
| 102 | request := release.RestartRequest{Scope: release.RestartScope{Kind: release.RestartScopeRetry, Value: rt.Host, Signed: wire}} |
| 103 | work := []scopedRestart{{app: [2]string{rt.Account, rt.App}, requests: []release.RestartRequest{request}}} |
| 104 | applyRetries([]registry.Route{rt}, work, &restartMemory{pub: pub}) |
| 105 | marker := filepath.Join(data, stageRequestMarker) |
| 106 | if _, err := os.Stat(marker); err != nil { |
| 107 | t.Fatal("root retry was not relayed", err) |
| 108 | } |
| 109 | if !book.HeldDown(rt.Account, rt.App, "v1", rt.Host) { |
| 110 | t.Fatal("agent relay cleared root authority") |
| 111 | } |
| 112 | if err := os.Remove(marker); err != nil { |
| 113 | t.Fatal(err) |
| 114 | } |
| 115 | work[0].requests[0].Scope.Signed = "forged" |
| 116 | applyRetries([]registry.Route{rt}, work, &restartMemory{pub: pub}) |
| 117 | if _, err := os.Stat(marker); !os.IsNotExist(err) { |
| 118 | t.Fatal("forged retry was relayed", err) |
| 119 | } |
| 120 | if err = book.VerifiedRetry(pub, wire, rt.Account, rt.App, rt.Host, time.Now()); err != nil { |
| 121 | t.Fatal(err) |
| 122 | } |
| 123 | if book.HeldDown(rt.Account, rt.App, "v1", rt.Host) { |
| 124 | t.Fatal("delivered retry did not clear root failure") |
| 125 | } |
| 126 | saved := rootUpdateRecord{Route: rt, RetryMirror: &rootLegacyRetry{Build: "v1", Book: legacy.ID(), Epoch: legacyEpoch}} |
| 127 | if err = p.publishRetry(saved, "v1"); err != nil { |
| 128 | t.Fatal(err) |
| 129 | } |
| 130 | local := &status.RolloutReport{Hosts: map[string]status.HostRollout{rt.Host: {Account: rt.Account, App: rt.App, HeldBy: "v1", HeldBook: legacy.ID(), HeldEpoch: legacyEpoch}}} |
| 131 | rootRetryStatus(local) |
| 132 | if legacy.HeldDown(rt.Account, rt.App, "v1", rt.Host) { |
| 133 | t.Fatal("agent mirror still blocks the root retry") |
| 134 | } |
| 135 | if _, err = legacy.Record(rollout.FailureRecord{Account: rt.Account, App: rt.App, Host: rt.Host, Build: "v1", HeldDown: []string{rt.Host}}); err != nil { |
| 136 | t.Fatal(err) |
| 137 | } |
| 138 | current := *local |
| 139 | current.Hosts = map[string]status.HostRollout{rt.Host: {HeldBy: "v1", HeldBook: legacy.ID(), HeldEpoch: legacy.PublishedEpoch(rt.Account, rt.App, "v1", rt.Host)}} |
| 140 | rootRetryStatus(¤t) |
| 141 | if !legacy.HeldDown(rt.Account, rt.App, "v1", rt.Host) { |
| 142 | t.Fatal("old receipt erased a later local failure") |
| 143 | } |
| 144 | // A service-written projection cannot deliver a mirror clear. |
| 145 | if err = os.Chmod(filepath.Join(rootRetryProjection, rt.Host+".json"), 0666); err != nil { |
| 146 | t.Fatal(err) |
| 147 | } |
| 148 | if report := rootRetryStatus(local); report == nil || report.Unavailable != rolloutBookUnreadable { |
| 149 | t.Fatal("hostile retry projection accepted", report) |
| 150 | } |
| 151 | |
| 152 | } |
| 153 | |
amadan