| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/instancekey" |
| 14 | "github.com/carlosframework/platform/internal/registry" |
| 15 | ) |
| 16 | |
| 17 | // The app the replica tests launch is this test binary, running |
| 18 | // TestInstanceKeyHelperProcess: rastrillo e3ea08de's carlos.InstanceKey |
| 19 | // and its documented caller, so a test sees what an adopting app does, not |
| 20 | // only which variables it was handed. |
| 21 | const helperOutEnv = "GO_TEST_INSTANCE_KEY_HELPER_OUT" |
| 22 | |
| 23 | var ( |
| 24 | errHelperNoInstanceKey = errors.New("ErrNoInstanceKey") |
| 25 | errHelperNotOnCarlos = errors.New("ErrNotOnCarlos") |
| 26 | ) |
| 27 | |
| 28 | // helperInstanceKey is rastrillo e3ea08de's InstanceKey(own) with Running(): |
| 29 | // own, then $CARLOS_INSTANCE_KEY verbatim, then ErrNoInstanceKey when |
| 30 | // $CARLOS_CONTROL_SOCKET is set, else ErrNotOnCarlos. |
| 31 | func helperInstanceKey(own string) (string, error) { |
| 32 | if own != "" { |
| 33 | return own, nil |
| 34 | } |
| 35 | if k := os.Getenv(InstanceKeyVar); k != "" { |
| 36 | return k, nil |
| 37 | } |
| 38 | if os.Getenv("CARLOS_CONTROL_SOCKET") != "" { |
| 39 | return "", errHelperNoInstanceKey |
| 40 | } |
| 41 | return "", errHelperNotOnCarlos |
| 42 | } |
| 43 | |
| 44 | // TestInstanceKeyHelperProcess is the app, not a test: with no output path |
| 45 | // set it does nothing. It resolves its key as an adopting app with no key |
| 46 | // of its own does, records the outcome, and exits: |
| 47 | // - "key <sha256>": it sealed with the key it was handed; |
| 48 | // - "refused": ErrNoInstanceKey, it did not start; |
| 49 | // - "minted": ErrNotOnCarlos, and the documented caller minted its own. |
| 50 | func TestInstanceKeyHelperProcess(t *testing.T) { |
| 51 | out := os.Getenv(helperOutEnv) |
| 52 | if out == "" { |
| 53 | return |
| 54 | } |
| 55 | var outcome string |
| 56 | switch key, err := helperInstanceKey(""); { |
| 57 | case err == nil: |
| 58 | outcome = "key " + instancekey.Digest(key) |
| 59 | case errors.Is(err, errHelperNoInstanceKey): |
| 60 | outcome = "refused" |
| 61 | default: |
| 62 | outcome = "minted" |
| 63 | } |
| 64 | _ = os.WriteFile(out+".tmp", []byte(outcome), 0o600) |
| 65 | _ = os.Rename(out+".tmp", out) |
| 66 | os.Exit(0) |
| 67 | } |
| 68 | |
| 69 | // stageHelperApp makes this test binary host's live build, running only |
| 70 | // TestInstanceKeyHelperProcess, and returns where its outcome lands. |
| 71 | func stageHelperApp(t *testing.T, b *ExecBackend, host, ver string) string { |
| 72 | t.Helper() |
| 73 | self, err := os.Executable() |
| 74 | if err != nil { |
| 75 | t.Fatal(err) |
| 76 | } |
| 77 | out := filepath.Join(t.TempDir(), "outcome") |
| 78 | t.Setenv(helperOutEnv, out) |
| 79 | vdir := filepath.Join(b.OptDir, "versions", ver) |
| 80 | if err := os.MkdirAll(vdir, 0o755); err != nil { |
| 81 | t.Fatal(err) |
| 82 | } |
| 83 | bin := filepath.Join(vdir, "app") |
| 84 | script := fmt.Sprintf("#!/bin/sh\nexec '%s' -test.run='^TestInstanceKeyHelperProcess$'\n", self) |
| 85 | if err := os.WriteFile(bin, []byte(script), 0o755); err != nil { |
| 86 | t.Fatal(err) |
| 87 | } |
| 88 | if err := os.MkdirAll(filepath.Join(b.OptDir, "live"), 0o755); err != nil { |
| 89 | t.Fatal(err) |
| 90 | } |
| 91 | os.Remove(b.liveLink(host)) |
| 92 | if err := os.Symlink(bin, b.liveLink(host)); err != nil { |
| 93 | t.Fatal(err) |
| 94 | } |
| 95 | return out |
| 96 | } |
| 97 | |
| 98 | // outcomeOf waits up to two seconds for the helper's outcome; "" is no app |
| 99 | // ran at all: none recorded, and no child of this test binary is left |
| 100 | // running that could (awaitChildFile). Call it once the launch returned. |
| 101 | func outcomeOf(path string) string { |
| 102 | return awaitChildFile(path, 2*time.Second) |
| 103 | } |
| 104 | |
| 105 | // noControlSocket is a socket path too long to bind, so the launch's |
| 106 | // control listener fails and CARLOS_CONTROL_SOCKET is withheld: the same |
| 107 | // launch a claim box runs before O6, and a flat launch whose bind failed. |
| 108 | func noControlSocket(t *testing.T) string { |
| 109 | return filepath.Join(t.TempDir(), strings.Repeat("s", 120)+".sock") |
| 110 | } |
| 111 | |
| 112 | type replicaCase struct { |
| 113 | name string |
| 114 | replica bool |
| 115 | own, config string // env file bodies ("" = no config file) |
| 116 | homeKey string // the home's effective key the box lists ("" = none listed) |
| 117 | start string // "refused" (no app runs) or the helper outcome wanted |
| 118 | wantKeyFromOwnOK bool // a home row: the outcome is the key minted into <host>.env |
| 119 | // bucket is the key the replica's configuration in the bucket renders |
| 120 | // now (ConfiguredKey; "" renders none). bucketDown makes that read |
| 121 | // fail; noBucket leaves ConfiguredKey nil. |
| 122 | bucket string |
| 123 | bucketDown, noBucket bool |
| 124 | } |
| 125 | |
| 126 | // freshConfig is ExecBackend.ConfiguredKey for one case. |
| 127 | func freshConfig(tc replicaCase) func(context.Context, registry.Route) (string, error) { |
| 128 | if tc.noBucket { |
| 129 | return nil |
| 130 | } |
| 131 | return func(context.Context, registry.Route) (string, error) { |
| 132 | switch { |
| 133 | case tc.bucketDown: |
| 134 | return "", errors.New("bucket unreachable") |
| 135 | case tc.bucket == "": |
| 136 | return "", nil |
| 137 | } |
| 138 | return instancekey.Digest(tc.bucket), nil |
| 139 | } |
| 140 | } |
| 141 | |
| 142 | // Rulings 26 to 28 through a real launch and the helper (Astra rounds 4 and |
| 143 | // 5): a replica runs only with exactly its home's key, listed by the |
| 144 | // pre-roll check and, when configured, still the bucket's configured key; |
| 145 | // or with none and a control socket its helper refuses on. Every launch |
| 146 | // here has no control socket, which is where the helper would otherwise |
| 147 | // mint. |
| 148 | func TestReplicaLaunchWithoutAControlSocket(t *testing.T) { |
| 149 | tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("c", 64) + "\n" |
| 150 | homeKey := strings.Repeat("H", 43) + "=" |
| 151 | for _, tc := range []replicaCase{ |
| 152 | {name: "a keyless replica is refused by the agent, not left to mint", replica: true, own: tok, start: "refused"}, |
| 153 | {name: "a configured key that is the home's runs and seals with it", replica: true, own: tok, config: InstanceKeyVar + "=" + homeKey + "\n", homeKey: homeKey, bucket: homeKey, start: "key " + instancekey.Digest(homeKey)}, |
| 154 | {name: "a pre-existing env key that is the home's runs", replica: true, own: tok + InstanceKeyVar + "=" + homeKey + "\n", homeKey: homeKey, start: "key " + instancekey.Digest(homeKey)}, |
| 155 | {name: "a configured key that differs is refused", replica: true, own: tok, config: InstanceKeyVar + "=configured-B\n", homeKey: homeKey, bucket: "configured-B", start: "refused"}, |
| 156 | {name: "a configured key with no verified home key is refused", replica: true, own: tok, config: InstanceKeyVar + "=" + homeKey + "\n", bucket: homeKey, start: "refused"}, |
| 157 | // Ruling 27, Astra round 5 finding 1: the key changed in the bucket |
| 158 | // to B and the home already holds it, while this box's sync is |
| 159 | // behind and the replica's configuration and list entry still say A. |
| 160 | {name: "the home updated, this replica's configuration delayed: refused", replica: true, own: tok, config: InstanceKeyVar + "=" + homeKey + "\n", homeKey: homeKey, bucket: "configured-B", start: "refused"}, |
| 161 | {name: "the bucket cannot be read: refused", replica: true, own: tok, config: InstanceKeyVar + "=" + homeKey + "\n", homeKey: homeKey, bucketDown: true, start: "refused"}, |
| 162 | {name: "no fresh reader: refused", replica: true, own: tok, config: InstanceKeyVar + "=" + homeKey + "\n", homeKey: homeKey, noBucket: true, start: "refused"}, |
| 163 | // Ruling 27's residual, documented: a change the bucket does not |
| 164 | // hold (the home's own <host>.env edited by hand, or a per-host key |
| 165 | // configured on the home only) is invisible here. The replica's |
| 166 | // list entry and the bucket both still say A, so it runs with A. |
| 167 | // Only the procedure, or O8's fresh evidence, closes this. |
| 168 | {name: "residual: a change only on the home is not seen, and the replica runs with the old key", replica: true, own: tok, config: InstanceKeyVar + "=" + homeKey + "\n", homeKey: homeKey, bucket: homeKey, start: "key " + instancekey.Digest(homeKey)}, |
| 169 | // Ruling 28: a home whose key exists only in configuration, with a |
| 170 | // replica that is handed none (its own overlay blanks it). O8 never |
| 171 | // delivers a configured key, so the replica stays keyless and is |
| 172 | // refused here, where it has no control socket. |
| 173 | {name: "a config-only home's keyless replica is refused", replica: true, own: tok, config: InstanceKeyVar + "=\n", homeKey: homeKey, bucket: "", start: "refused"}, |
| 174 | {name: "a home row mints and its app seals with the minted key", own: tok, wantKeyFromOwnOK: true}, |
| 175 | } { |
| 176 | t.Run(tc.name, func(t *testing.T) { |
| 177 | b := testExecBackend(t) |
| 178 | rt := registry.Route{Host: "shop-au.example.com", Kind: "instance", Replica: tc.replica, Socket: noControlSocket(t)} |
| 179 | own := filepath.Join(b.DataDir, rt.Host+".env") |
| 180 | if err := os.WriteFile(own, []byte(tc.own), 0o600); err != nil { |
| 181 | t.Fatal(err) |
| 182 | } |
| 183 | if tc.config != "" { |
| 184 | if err := os.WriteFile(filepath.Join(b.DataDir, rt.Host+".config.env"), []byte(tc.config), 0o600); err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | } |
| 188 | if tc.homeKey != "" { |
| 189 | b.ReplicaHomeKeys = func(h string) (string, bool) { |
| 190 | return instancekey.Digest(tc.homeKey), h == rt.Host |
| 191 | } |
| 192 | } |
| 193 | b.ConfiguredKey = freshConfig(tc) |
| 194 | out := stageHelperApp(t, b, rt.Host, "v1") |
| 195 | ctx, cancel := recordedCtx(2*time.Second, out) |
| 196 | defer cancel() |
| 197 | proc, _, err := b.Start(ctx, rt) |
| 198 | if proc != nil { |
| 199 | defer proc.Stop(context.Background()) |
| 200 | } |
| 201 | got := outcomeOf(out) |
| 202 | if tc.wantKeyFromOwnOK { |
| 203 | env, _ := readEnvFile(own) |
| 204 | if got != "key "+instancekey.Digest(env[InstanceKeyVar]) { |
| 205 | t.Fatalf("home row: outcome %q, want the minted key", got) |
| 206 | } |
| 207 | return |
| 208 | } |
| 209 | if tc.start == "refused" { |
| 210 | if !errors.Is(err, ErrReplicaKey) || got != "" { |
| 211 | t.Fatalf("Start = %v, app outcome %q; want ErrReplicaKey and no app", err, got) |
| 212 | } |
| 213 | } else if got != tc.start { |
| 214 | t.Fatalf("app outcome %q, want %q (Start: %v)", got, tc.start, err) |
| 215 | } |
| 216 | if raw, _ := os.ReadFile(own); string(raw) != tc.own { |
| 217 | t.Fatal("a replica launch changed its own env file: a replica never mints") |
| 218 | } |
| 219 | }) |
| 220 | } |
| 221 | } |
| 222 | |
| 223 | // A replica's seed has no control socket: a keyless replica's seed is |
| 224 | // refused before it runs; one handed its home's key runs. |
| 225 | func TestReplicaSeedIsGated(t *testing.T) { |
| 226 | tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("c", 64) + "\n" |
| 227 | homeKey := strings.Repeat("H", 43) + "=" |
| 228 | for _, tc := range []struct { |
| 229 | name, config string |
| 230 | want string |
| 231 | }{ |
| 232 | {"keyless", "", ""}, |
| 233 | {"the home's key", InstanceKeyVar + "=" + homeKey + "\n", "key " + instancekey.Digest(homeKey)}, |
| 234 | } { |
| 235 | t.Run(tc.name, func(t *testing.T) { |
| 236 | b := testExecBackend(t) |
| 237 | rt := registry.Route{Host: "shop-au.example.com", Kind: "instance", Replica: true, Seed: `["seed"]`} |
| 238 | if err := os.WriteFile(filepath.Join(b.DataDir, rt.Host+".env"), []byte(tok), 0o600); err != nil { |
| 239 | t.Fatal(err) |
| 240 | } |
| 241 | if tc.config != "" { |
| 242 | if err := os.WriteFile(filepath.Join(b.DataDir, rt.Host+".config.env"), []byte(tc.config), 0o600); err != nil { |
| 243 | t.Fatal(err) |
| 244 | } |
| 245 | } |
| 246 | b.ReplicaHomeKeys = func(h string) (string, bool) { return instancekey.Digest(homeKey), true } |
| 247 | b.ConfiguredKey = freshConfig(replicaCase{bucket: homeKey}) |
| 248 | out := stageHelperApp(t, b, rt.Host, "v1") |
| 249 | err := b.Seed(context.Background(), rt) |
| 250 | got := outcomeOf(out) |
| 251 | if tc.want == "" { |
| 252 | if !errors.Is(err, ErrReplicaKey) || got != "" { |
| 253 | t.Fatalf("Seed = %v, outcome %q; want ErrReplicaKey and no seed process", err, got) |
| 254 | } |
| 255 | return |
| 256 | } |
| 257 | if err != nil || got != tc.want { |
| 258 | t.Fatalf("Seed = %v, outcome %q; want %q", err, got, tc.want) |
| 259 | } |
| 260 | }) |
| 261 | } |
| 262 | } |
| 263 | |
| 264 | // A worker (StartWorkerStaged) has no control socket, so a replica row's |
| 265 | // worker is gated as its seed is (ruling 26), on the layer from the same |
| 266 | // read: it runs only with its home's key, verified, and is refused with a |
| 267 | // key that is not the home's or with none at all. A worker never mints |
| 268 | // (ruling 12), and a worker on a row that is not a replica is unaffected. |
| 269 | func TestReplicaWorkerIsGatedLikeASeed(t *testing.T) { |
| 270 | tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("c", 64) + "\n" |
| 271 | homeKey := strings.Repeat("H", 43) + "=" |
| 272 | for _, tc := range []struct { |
| 273 | name string |
| 274 | replica bool |
| 275 | config string // <host>.config.env ("" = none) |
| 276 | bucket string // the key the bucket's configuration renders now |
| 277 | want string // the helper's outcome; "" = refused, no worker ran |
| 278 | // extraKey hands the home's key through extra as well as the env |
| 279 | // files: a key whose origin the read cannot vouch for. |
| 280 | extraKey bool |
| 281 | }{ |
| 282 | {"a replica worker with the home's key runs", true, InstanceKeyVar + "=" + homeKey + "\n", homeKey, "key " + instancekey.Digest(homeKey), false}, |
| 283 | // The bucket renders configured-B too, so only the home-key digest |
| 284 | // check can refuse this one. |
| 285 | {"a replica worker with a different configured key is refused", true, InstanceKeyVar + "=configured-B\n", "configured-B", "", false}, |
| 286 | {"a keyless replica worker is refused", true, "", "", "", false}, |
| 287 | {"a worker on a row that is not a replica is unaffected", false, InstanceKeyVar + "=configured-B\n", "", "key " + instancekey.Digest("configured-B"), false}, |
| 288 | {"a replica worker handed a key through extra is refused", true, InstanceKeyVar + "=" + homeKey + "\n", homeKey, "", true}, |
| 289 | } { |
| 290 | t.Run(tc.name, func(t *testing.T) { |
| 291 | b := testExecBackend(t) |
| 292 | rt := registry.Route{Host: "shop-au.example.com", Kind: "sidecar", Args: `["sidecar","run"]`, Replica: tc.replica} |
| 293 | own := filepath.Join(b.DataDir, rt.Host+".env") |
| 294 | if err := os.WriteFile(own, []byte(tok), 0o600); err != nil { |
| 295 | t.Fatal(err) |
| 296 | } |
| 297 | if tc.config != "" { |
| 298 | if err := os.WriteFile(filepath.Join(b.DataDir, rt.Host+".config.env"), []byte(tc.config), 0o600); err != nil { |
| 299 | t.Fatal(err) |
| 300 | } |
| 301 | } |
| 302 | if tc.replica { |
| 303 | b.ReplicaHomeKeys = func(h string) (string, bool) { return instancekey.Digest(homeKey), h == rt.Host } |
| 304 | b.ConfiguredKey = freshConfig(replicaCase{bucket: tc.bucket}) |
| 305 | } |
| 306 | out := stageHelperApp(t, b, rt.Host, "v1") |
| 307 | extra := map[string]string{"CARLOS_SIDECAR_NAME": "bot"} |
| 308 | if tc.extraKey { |
| 309 | extra[InstanceKeyVar] = homeKey |
| 310 | } |
| 311 | proc, _, err := b.StartWorkerStaged(context.Background(), rt, extra, nil) |
| 312 | if proc != nil { |
| 313 | defer proc.Stop(context.Background()) |
| 314 | } |
| 315 | got := outcomeOf(out) |
| 316 | if tc.want == "" { |
| 317 | if !errors.Is(err, ErrReplicaKey) || proc != nil || got != "" { |
| 318 | t.Fatalf("StartWorkerStaged = %v (proc %v), outcome %q; want ErrReplicaKey and no worker", err, proc != nil, got) |
| 319 | } |
| 320 | } else if err != nil || got != tc.want { |
| 321 | t.Fatalf("StartWorkerStaged = %v, outcome %q; want %q", err, got, tc.want) |
| 322 | } |
| 323 | if raw, _ := os.ReadFile(own); string(raw) != tok { |
| 324 | t.Fatal("a worker launch changed its own env file: a worker never mints") |
| 325 | } |
| 326 | }) |
| 327 | } |
| 328 | } |
| 329 | |
| 330 | // The legacy co-spawned sidecar (<host>-sidecar.env, StartSidecar) runs the |
| 331 | // app binary with no control socket, so on a replica host it is gated as |
| 332 | // the worker is (ruling 26): the key in its own file, if any, must be the |
| 333 | // home's verified key, and a keyless one is refused rather than left to |
| 334 | // mint. A sidecar on a host that is not a replica is unaffected. |
| 335 | func TestReplicaLegacySidecarIsGatedLikeTheWorker(t *testing.T) { |
| 336 | homeKey := strings.Repeat("H", 43) + "=" |
| 337 | for _, tc := range []struct { |
| 338 | name string |
| 339 | replica bool |
| 340 | sidecar string // <host>-sidecar.env |
| 341 | want string // the helper's outcome; "" = refused, no sidecar ran |
| 342 | }{ |
| 343 | {"a replica's sidecar with the home's key runs", true, InstanceKeyVar + "=" + homeKey + "\n", "key " + instancekey.Digest(homeKey)}, |
| 344 | {"a replica's sidecar with another key is refused", true, InstanceKeyVar + "=configured-B\n", ""}, |
| 345 | {"a replica's keyless sidecar is refused", true, "OTHER=1\n", ""}, |
| 346 | {"a sidecar on a host that is not a replica is unaffected", false, InstanceKeyVar + "=configured-B\n", "key " + instancekey.Digest("configured-B")}, |
| 347 | } { |
| 348 | t.Run(tc.name, func(t *testing.T) { |
| 349 | b := testExecBackend(t) |
| 350 | rt := registry.Route{Host: "shop-au.example.com", Kind: "instance", Replica: tc.replica} |
| 351 | sidecarEnv := filepath.Join(b.DataDir, rt.Host+"-sidecar.env") |
| 352 | if err := os.WriteFile(sidecarEnv, []byte(tc.sidecar), 0o600); err != nil { |
| 353 | t.Fatal(err) |
| 354 | } |
| 355 | if tc.replica { |
| 356 | b.ReplicaHomeKeys = func(h string) (string, bool) { return instancekey.Digest(homeKey), h == rt.Host } |
| 357 | // The bucket agrees with whatever the case holds, so only the |
| 358 | // home-key check decides. |
| 359 | b.ConfiguredKey = freshConfig(replicaCase{bucket: "configured-B"}) |
| 360 | } |
| 361 | out := stageHelperApp(t, b, rt.Host, "v1") |
| 362 | proc, err := b.StartSidecar(context.Background(), rt) |
| 363 | if proc != nil { |
| 364 | defer proc.Stop(context.Background()) |
| 365 | } |
| 366 | got := outcomeOf(out) |
| 367 | if tc.want == "" { |
| 368 | if !errors.Is(err, ErrReplicaKey) || proc != nil || got != "" { |
| 369 | t.Fatalf("StartSidecar = %v (proc %v), outcome %q; want ErrReplicaKey and no sidecar", err, proc != nil, got) |
| 370 | } |
| 371 | } else if err != nil || got != tc.want { |
| 372 | t.Fatalf("StartSidecar = %v, outcome %q; want %q", err, got, tc.want) |
| 373 | } |
| 374 | if raw, _ := os.ReadFile(sidecarEnv); string(raw) != tc.sidecar { |
| 375 | t.Fatal("a sidecar launch changed its env file: nothing mints there") |
| 376 | } |
| 377 | }) |
| 378 | } |
| 379 | } |
| 380 | |
| 381 | // Two guards that a later check would otherwise back up: with no list entry |
| 382 | // for the replica, or a bucket read that fails, the launch is refused for |
| 383 | // that reason, named, and not merely because a digest compared against "" |
| 384 | // failed. The refusal names the file an operator must fix. |
| 385 | func TestReplicaGateNamesWhyItRefused(t *testing.T) { |
| 386 | keyA := strings.Repeat("A", 43) + "=" |
| 387 | rt := registry.Route{Host: "shop-au.example.com", Kind: "instance", Replica: true} |
| 388 | env := map[string]string{InstanceKeyVar: keyA} |
| 389 | listed := func(string) (string, bool) { return instancekey.Digest(keyA), true } |
| 390 | bucketA := func(context.Context, registry.Route) (string, error) { return instancekey.Digest(keyA), nil } |
| 391 | for _, tc := range []struct { |
| 392 | name string |
| 393 | homes func(string) (string, bool) |
| 394 | bucket func(context.Context, registry.Route) (string, error) |
| 395 | want string |
| 396 | }{ |
| 397 | {"no entry in the list", func(string) (string, bool) { return "", false }, bucketA, "no entry for it in /etc/carlos/replica-home-keys"}, |
| 398 | {"no list reader at all", nil, bucketA, "no entry for it in /etc/carlos/replica-home-keys"}, |
| 399 | {"the bucket read fails", listed, func(context.Context, registry.Route) (string, error) { |
| 400 | return "", errors.New("bucket unreachable") |
| 401 | }, "the bucket's configuration could not be read: bucket unreachable"}, |
| 402 | } { |
| 403 | t.Run(tc.name, func(t *testing.T) { |
| 404 | b := &ExecBackend{DataDir: t.TempDir(), ReplicaHomeKeys: tc.homes, ConfiguredKey: tc.bucket} |
| 405 | err := b.replicaKeyGate(rt, env, instancekey.LayerConfig, false) |
| 406 | if !errors.Is(err, ErrReplicaKey) || !strings.Contains(err.Error(), tc.want) { |
| 407 | t.Fatalf("gate = %v; want ErrReplicaKey naming %q", err, tc.want) |
| 408 | } |
| 409 | }) |
| 410 | } |
| 411 | } |
| 412 | |
| 413 | // Final review I5 and minor 5: the gate's refusal is logged once an hour |
| 414 | // per host and reason, so a keyless replica's legacy sidecar, refused at |
| 415 | // every retry of the sidecar sweep, does not fill the log; a new reason or |
| 416 | // another host is still heard. A bucket error is logged apart from the |
| 417 | // fixed reason. |
| 418 | func TestReplicaGateLogsARefusalOncePerHostAndReason(t *testing.T) { |
| 419 | buf := captureLog(t) |
| 420 | b := &ExecBackend{DataDir: t.TempDir()} |
| 421 | rt := registry.Route{Host: "ratelimit-au.example.com", Kind: "instance", Replica: true} |
| 422 | for i := 0; i < 3; i++ { |
| 423 | if err := b.replicaKeyGate(rt, map[string]string{}, instancekey.LayerNone, false); !errors.Is(err, ErrReplicaKey) { |
| 424 | t.Fatalf("gate %d = %v", i, err) |
| 425 | } |
| 426 | } |
| 427 | other := rt |
| 428 | other.Host = "ratelimit-eu.example.com" |
| 429 | if err := b.replicaKeyGate(other, map[string]string{}, instancekey.LayerNone, false); !errors.Is(err, ErrReplicaKey) { |
| 430 | t.Fatal(err) |
| 431 | } |
| 432 | b.ReplicaHomeKeys = func(string) (string, bool) { return instancekey.Digest("home"), true } |
| 433 | b.ConfiguredKey = func(context.Context, registry.Route) (string, error) { return "", errors.New("bucket unreachable") } |
| 434 | env := map[string]string{instancekey.Var: "home"} |
| 435 | for i := 0; i < 2; i++ { |
| 436 | if err := b.replicaKeyGate(rt, env, instancekey.LayerConfig, false); !errors.Is(err, ErrReplicaKey) || !strings.Contains(err.Error(), "bucket unreachable") { |
| 437 | t.Fatalf("gate with the bucket down = %v", err) |
| 438 | } |
| 439 | } |
| 440 | lines := strings.Split(strings.TrimSpace(buf.String()), "\n") |
| 441 | var au, eu, bucket int |
| 442 | for _, l := range lines { |
| 443 | switch { |
| 444 | case strings.Contains(l, "replica ratelimit-au.example.com not started: the bucket's configuration could not be read: bucket unreachable"): |
| 445 | bucket++ |
| 446 | case strings.Contains(l, "replica ratelimit-au.example.com not started"): |
| 447 | au++ |
| 448 | case strings.Contains(l, "replica ratelimit-eu.example.com not started"): |
| 449 | eu++ |
| 450 | } |
| 451 | } |
| 452 | if au != 1 || eu != 1 || bucket != 1 { |
| 453 | t.Fatalf("refusal lines: au %d, eu %d, bucket %d; want one each:\n%s", au, eu, bucket, buf.String()) |
| 454 | } |
| 455 | } |
| 456 | |
amadan