| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "os" |
| 9 | "path/filepath" |
| 10 | "testing" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/hostbox" |
| 13 | "github.com/carlosframework/platform/internal/hostclaim" |
| 14 | "github.com/carlosframework/platform/internal/hostwrite" |
| 15 | "github.com/carlosframework/platform/internal/provision" |
| 16 | "github.com/carlosframework/platform/internal/registry" |
| 17 | "github.com/carlosframework/platform/internal/release" |
| 18 | "github.com/carlosframework/platform/internal/rootstate" |
| 19 | ) |
| 20 | |
| 21 | // Task 7b, the agent's side: under the claim with root's relay wired, T7's |
| 22 | // blocked stage and every publication are asked of root (movesstage.go, |
| 23 | // heldReleaseHooksVia; the probe's heldReleaseProbeRelease), and the agent |
| 24 | // passes only on its own reading afterwards. Root's side and the fixture |
| 25 | // are in movesheldrelease_test.go. |
| 26 | |
| 27 | // heldReleaseRelayFunc is a relay that calls root directly, as the socket |
| 28 | // transport would after authenticating the agent. |
| 29 | type heldReleaseRelayFunc func(context.Context, heldReleaseRequest) (heldReleaseResponse, error) |
| 30 | |
| 31 | func (f heldReleaseRelayFunc) HeldRelease(ctx context.Context, q heldReleaseRequest) (heldReleaseResponse, error) { |
| 32 | return f(ctx, q) |
| 33 | } |
| 34 | |
| 35 | func (f *hrFixture) relay(seen *[]heldReleaseRequest) heldReleaseRelay { |
| 36 | return heldReleaseRelayFunc(func(ctx context.Context, q heldReleaseRequest) (heldReleaseResponse, error) { |
| 37 | if seen != nil { |
| 38 | *seen = append(*seen, q) |
| 39 | } |
| 40 | return f.root.Serve(ctx, hrPeer, q) |
| 41 | }) |
| 42 | } |
| 43 | |
| 44 | // phase1Root is root on the target box of a Phase 1 harness, over the |
| 45 | // harness's own claim, record and source report. |
| 46 | func phase1Root(t *testing.T, h *phase1Harness, opt, data string) *movesHeldReleaseRoot { |
| 47 | t.Helper() |
| 48 | return &movesHeldReleaseRoot{ |
| 49 | OpenRelease: func() (rootstate.Store, error) { return &plan14ReleaseStore{}, nil }, |
| 50 | Actor: func(context.Context) (hostbox.Actor, error) { |
| 51 | return hostbox.Actor{Box: "uk-box", Region: "uk", Eligible: true}, nil |
| 52 | }, |
| 53 | Store: h.s, |
| 54 | Claims: func(ctx context.Context, host string) (hostclaim.Observed, error) { |
| 55 | return hostclaim.Read(ctx, h.s, host) |
| 56 | }, |
| 57 | Records: MovesRecordsFrom(h.s), |
| 58 | Route: func(string) (*registry.Route, error) { return nil, nil }, |
| 59 | Works: func(context.Context, hostbox.Actor, string, string) (hostwrite.Work, error) { |
| 60 | t.Fatal("a stage selected") |
| 61 | return nil, nil |
| 62 | }, |
| 63 | Selected: hostbox.Selected, |
| 64 | Pub: hsPub, |
| 65 | Ratchet: func() *release.DigestRatchet { return hsRatchet(t) }, |
| 66 | Proof: func() *release.ProofGate { return offProofGate(t) }, |
| 67 | OptDir: opt, |
| 68 | DataDir: data, |
| 69 | ReceiptDir: t.TempDir(), |
| 70 | } |
| 71 | } |
| 72 | |
| 73 | // blockedOnce makes the agent's version store read-only, as on a hardened |
| 74 | // box: a stage that would write fails as EROFS does, and a version already |
| 75 | // staged (by root) is read and verified as release.StageHeldVerified reads |
| 76 | // it. |
| 77 | func blockedOnce(t *testing.T) { |
| 78 | t.Helper() |
| 79 | prev := heldStageVerified |
| 80 | t.Cleanup(func() { heldStageVerified = prev }) |
| 81 | heldStageVerified = func(ctx context.Context, s release.Store, pub ed25519.PublicKey, r *release.DigestRatchet, optDir string, rt registry.Route, version string, proof *release.ProofGate) (*release.Manifest, error) { |
| 82 | if _, err := os.Stat(release.VersionPath(optDir, release.RouteAccount(rt), version, rt.App)); err != nil { |
| 83 | return nil, fmt.Errorf("%w: read-only file system", release.ErrStageBlocked) |
| 84 | } |
| 85 | return prev(ctx, s, pub, r, optDir, rt, version, proof) |
| 86 | } |
| 87 | } |
| 88 | |
| 89 | // TestHeldReleaseStagesThroughRootWithoutPublishing: on a hardened box the |
| 90 | // agent's stage is EROFS; under the claim T7 asks root to stage, root |
| 91 | // stages and verifies without publishing, and T7 passes on the agent's own |
| 92 | // re-verification of the staged bytes. Nothing is left for the old file |
| 93 | // relay, and nothing is selected. |
| 94 | func TestHeldReleaseStagesThroughRootWithoutPublishing(t *testing.T) { |
| 95 | ctx := context.Background() |
| 96 | h := newPhase1Harness(t) |
| 97 | rec := h.start(t, "manual") |
| 98 | shipPinnedV7(t, h.s) |
| 99 | h.pass(t) |
| 100 | t.Setenv("CARLOS_HOST_CLAIM", "1") |
| 101 | opt, data := t.TempDir(), t.TempDir() |
| 102 | root := phase1Root(t, h, opt, data) |
| 103 | blockedOnce(t) |
| 104 | var seen []heldReleaseRequest |
| 105 | relay := heldReleaseRelayFunc(func(ctx context.Context, q heldReleaseRequest) (heldReleaseResponse, error) { |
| 106 | seen = append(seen, q) |
| 107 | return root.Serve(ctx, hrPeer, q) |
| 108 | }) |
| 109 | sel := &heldSelections{} |
| 110 | relayedFile := false |
| 111 | ready, _ := heldReleaseHooksVia(heldReleaseVia{Relay: relay, Region: "uk"}, claimWrites(sel), testRegistry(t), opt, data, "uk-box", |
| 112 | hsPub, hsRatchet(t), offProofGate(t), func(string, string, error) { relayedFile = true }) |
| 113 | if c := ready(ctx, h.s, rec, "v7"); !c.OK { |
| 114 | t.Fatalf("T7 through root = %+v", c) |
| 115 | } |
| 116 | if len(seen) != 1 || seen[0].Action != heldReleaseStage || seen[0].Version != "v7" || seen[0].Kind != hostclaim.KindMoving || seen[0].RenderHash != "" { |
| 117 | t.Fatalf("root was asked %+v", seen) |
| 118 | } |
| 119 | if _, err := os.Stat(release.VersionPath(opt, "bab", "v7", "app")); err != nil { |
| 120 | t.Fatalf("v7 not staged: %v", err) |
| 121 | } |
| 122 | if _, err := os.Lstat(release.LiveLink(opt, p1Host)); !errors.Is(err, os.ErrNotExist) { |
| 123 | t.Fatalf("root's stage published live/<host>: %v", err) |
| 124 | } |
| 125 | if _, err := os.Stat(release.HeldStageDir(data)); !errors.Is(err, os.ErrNotExist) || relayedFile { |
| 126 | t.Fatalf("a file request was left for the old relay (%v, %v)", err, relayedFile) |
| 127 | } |
| 128 | if len(sel.selected) != 0 { |
| 129 | t.Fatalf("T7 selected %v", sel.selected) |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | // TestHeldReleaseConfiguredPublishesThroughRoot: under the claim with root's |
| 134 | // relay wired, configured's release hook never publishes in the agent: it |
| 135 | // asks root with the claim's tuple and the journalled render hash, and then |
| 136 | // passes on its own reading. A replay finds the release live and asks |
| 137 | // nothing. |
| 138 | func TestHeldReleaseConfiguredPublishesThroughRoot(t *testing.T) { |
| 139 | ctx := context.Background() |
| 140 | t.Setenv("CARLOS_HOST_CLAIM", "1") |
| 141 | f := newHRFixture(t) |
| 142 | var seen []heldReleaseRequest |
| 143 | agentSel := &heldSelections{} |
| 144 | _, live := heldReleaseHooksVia(heldReleaseVia{Relay: f.relay(&seen), Region: "uk", Claims: f.claims}, claimWrites(agentSel), testRegistry(t), f.opt, f.data, "uk-box", |
| 145 | hsPub, hsRatchet(t), offProofGate(t), nil) |
| 146 | if c := live(ctx, f.s, f.rec, "v7"); !c.OK { |
| 147 | t.Fatalf("configured's release through root = %+v", c) |
| 148 | } |
| 149 | if len(seen) != 1 { |
| 150 | t.Fatalf("root was asked %d times", len(seen)) |
| 151 | } |
| 152 | q := seen[0] |
| 153 | if q.Action != heldReleasePublish || q.RenderHash != hrRendered || q.HoldOp != p1ID || q.MoveID != p1ID || q.Stamp != 7 || q.Session != 3 || |
| 154 | q.Generation != p1ID || q.Kind != hostclaim.KindLanding || q.Phase != provision.LandConfigured || q.Region != "uk" || q.Box != "uk-box" { |
| 155 | t.Fatalf("the agent's request = %+v", q) |
| 156 | } |
| 157 | if len(agentSel.selected) != 0 { |
| 158 | t.Fatalf("the agent selected %v itself", agentSel.selected) |
| 159 | } |
| 160 | if v, _ := release.LiveVersion(f.opt, p1Host); v != "v7" { |
| 161 | t.Fatalf("live = %q", v) |
| 162 | } |
| 163 | if c := live(ctx, f.s, f.rec, "v7"); !c.OK || len(seen) != 1 { |
| 164 | t.Fatalf("a replay: %+v, asked %d times", c, len(seen)) |
| 165 | } |
| 166 | |
| 167 | t.Run("not yet waits for the next pass", func(t *testing.T) { |
| 168 | g := newHRFixture(t) |
| 169 | g.root.DataDir = t.TempDir() |
| 170 | _, live := heldReleaseHooksVia(heldReleaseVia{Relay: g.relay(nil), Region: "uk", Claims: g.claims}, claimWrites(&heldSelections{}), testRegistry(t), g.opt, g.data, "uk-box", |
| 171 | hsPub, hsRatchet(t), offProofGate(t), nil) |
| 172 | if c := live(ctx, g.s, g.rec, "v7"); c.OK { |
| 173 | t.Fatalf("published with root's journal missing: %+v", c) |
| 174 | } |
| 175 | if v, _ := release.LiveVersion(g.opt, p1Host); v != "" { |
| 176 | t.Fatalf("live = %q", v) |
| 177 | } |
| 178 | }) |
| 179 | } |
| 180 | |
| 181 | // TestHeldReleaseProbeReleaseAdapter: the probe's release adapter publishes |
| 182 | // through root when live/<host> does not name the pinned release, and asks |
| 183 | // nothing when it does. |
| 184 | func TestHeldReleaseProbeReleaseAdapter(t *testing.T) { |
| 185 | ctx := context.Background() |
| 186 | f := newHRFixture(t) |
| 187 | f.set(func(f *hrFixture) { |
| 188 | f.claim.Op.Phase = provision.LandProbing |
| 189 | f.rec.Move.LandingPhase = provision.LandProbing |
| 190 | }) |
| 191 | var seen []heldReleaseRequest |
| 192 | adapter := heldReleaseProbeRelease(heldReleaseVia{Relay: f.relay(&seen), Region: "uk", Claims: f.claims}, f.s, "uk-box", f.opt, f.data, offProofGate(t)) |
| 193 | rt := registry.Route{Host: p1Host, Kind: "instance", App: "app", Account: "bab", HoldOp: p1ID} |
| 194 | if err := adapter(ctx, rt, "v7", heldSelectionWork{&heldSelections{}}); err != nil { |
| 195 | t.Fatalf("the probe's release: %v", err) |
| 196 | } |
| 197 | if len(seen) != 1 || seen[0].Phase != provision.LandProbing || seen[0].Action != heldReleasePublish { |
| 198 | t.Fatalf("root was asked %+v", seen) |
| 199 | } |
| 200 | if err := adapter(ctx, rt, "v7", heldSelectionWork{&heldSelections{}}); err != nil || len(seen) != 1 { |
| 201 | t.Fatalf("a live release: %v, asked %d times", err, len(seen)) |
| 202 | } |
| 203 | } |
| 204 | |
| 205 | // TestHeldReleaseOldConsumers: a root that has no held-release route (an |
| 206 | // older build, or the claim off) answers unavailable, and T7 says MV-N455 |
| 207 | // with nothing staged, published or left as a file; and the old file relay, |
| 208 | // run after the new path staged, finds no request and publishes nothing. |
| 209 | func TestHeldReleaseOldConsumers(t *testing.T) { |
| 210 | ctx := context.Background() |
| 211 | |
| 212 | t.Run("an old root", func(t *testing.T) { |
| 213 | h := newPhase1Harness(t) |
| 214 | rec := h.start(t, "manual") |
| 215 | shipPinnedV7(t, h.s) |
| 216 | h.pass(t) |
| 217 | t.Setenv("CARLOS_HOST_CLAIM", "1") |
| 218 | opt, data := t.TempDir(), t.TempDir() |
| 219 | blockedOnce(t) |
| 220 | old := heldReleaseRelayFunc(func(context.Context, heldReleaseRequest) (heldReleaseResponse, error) { |
| 221 | return heldReleaseResponse{}, errHeldReleaseUnavailable |
| 222 | }) |
| 223 | ready, _ := heldReleaseHooksVia(heldReleaseVia{Relay: old, Region: "uk"}, claimWrites(&heldSelections{}), testRegistry(t), opt, data, "uk-box", |
| 224 | hsPub, hsRatchet(t), offProofGate(t), nil) |
| 225 | if c := ready(ctx, h.s, rec, "v7"); c.OK || c.Reason != t7NotStaged { |
| 226 | t.Fatalf("T7 against an old root = %+v, want %q", c, t7NotStaged) |
| 227 | } |
| 228 | if _, err := os.Stat(filepath.Join(opt, "versions")); !errors.Is(err, os.ErrNotExist) { |
| 229 | t.Fatalf("staged: %v", err) |
| 230 | } |
| 231 | if _, err := os.Stat(release.HeldStageDir(data)); !errors.Is(err, os.ErrNotExist) { |
| 232 | t.Fatalf("a file request was left for the old relay: %v", err) |
| 233 | } |
| 234 | }) |
| 235 | |
| 236 | t.Run("the old file relay publishes nothing", func(t *testing.T) { |
| 237 | h := newPhase1Harness(t) |
| 238 | rec := h.start(t, "manual") |
| 239 | shipPinnedV7(t, h.s) |
| 240 | h.pass(t) |
| 241 | t.Setenv("CARLOS_HOST_CLAIM", "1") |
| 242 | opt, data := t.TempDir(), t.TempDir() |
| 243 | root := phase1Root(t, h, opt, data) |
| 244 | blockedOnce(t) |
| 245 | ready, _ := heldReleaseHooksVia(heldReleaseVia{Relay: heldReleaseRelayFunc(func(ctx context.Context, q heldReleaseRequest) (heldReleaseResponse, error) { |
| 246 | return root.Serve(ctx, hrPeer, q) |
| 247 | }), Region: "uk"}, claimWrites(&heldSelections{}), testRegistry(t), opt, data, "uk-box", hsPub, hsRatchet(t), offProofGate(t), nil) |
| 248 | if c := ready(ctx, h.s, rec, "v7"); !c.OK { |
| 249 | t.Fatalf("T7 = %+v", c) |
| 250 | } |
| 251 | t.Setenv("CARLOS_HOST_CLAIM", "") |
| 252 | authorized := false |
| 253 | if err := release.ServeHeldStageRequestsWithPins(ctx, h.s, hsPub, hsRatchet(t), opt, data, offProofGate(t), func(release.HeldStageRequest, func() error) error { |
| 254 | authorized = true |
| 255 | return nil |
| 256 | }); err != nil { |
| 257 | t.Fatal(err) |
| 258 | } |
| 259 | if authorized { |
| 260 | t.Fatal("the old relay found a request") |
| 261 | } |
| 262 | if _, err := os.Lstat(release.LiveLink(opt, p1Host)); !errors.Is(err, os.ErrNotExist) { |
| 263 | t.Fatalf("the old relay published: %v", err) |
| 264 | } |
| 265 | }) |
| 266 | } |
| 267 | |
| 268 | // TestHeldReleaseAgentVerifiesRootsStage: root's answer to a stage is a |
| 269 | // hint. T7 passes only on the agent's own reading of the staged bytes, so a |
| 270 | // root that answers without staging never passes T7, and a stage whose |
| 271 | // answer was lost passes on the next pass's reading without asking root |
| 272 | // again. |
| 273 | func TestHeldReleaseAgentVerifiesRootsStage(t *testing.T) { |
| 274 | ctx := context.Background() |
| 275 | |
| 276 | t.Run("root's answer is a hint", func(t *testing.T) { |
| 277 | h := newPhase1Harness(t) |
| 278 | rec := h.start(t, "manual") |
| 279 | shipPinnedV7(t, h.s) |
| 280 | h.pass(t) |
| 281 | t.Setenv("CARLOS_HOST_CLAIM", "1") |
| 282 | opt, data := t.TempDir(), t.TempDir() |
| 283 | blockedOnce(t) |
| 284 | lying := heldReleaseRelayFunc(func(_ context.Context, q heldReleaseRequest) (heldReleaseResponse, error) { |
| 285 | return heldReleaseResponse{Schema: heldReleaseSchema, RequestID: q.RequestID, Outcome: heldReleaseOutcomeStaged}, nil |
| 286 | }) |
| 287 | ready, _ := heldReleaseHooksVia(heldReleaseVia{Relay: lying, Region: "uk"}, claimWrites(&heldSelections{}), testRegistry(t), opt, data, "uk-box", |
| 288 | hsPub, hsRatchet(t), offProofGate(t), nil) |
| 289 | if c := ready(ctx, h.s, rec, "v7"); c.OK || c.Reason != t7NotStaged { |
| 290 | t.Fatalf("T7 on root's word alone = %+v, want %q", c, t7NotStaged) |
| 291 | } |
| 292 | }) |
| 293 | |
| 294 | t.Run("a stage whose answer is lost", func(t *testing.T) { |
| 295 | h := newPhase1Harness(t) |
| 296 | rec := h.start(t, "manual") |
| 297 | shipPinnedV7(t, h.s) |
| 298 | h.pass(t) |
| 299 | t.Setenv("CARLOS_HOST_CLAIM", "1") |
| 300 | opt, data := t.TempDir(), t.TempDir() |
| 301 | root := phase1Root(t, h, opt, data) |
| 302 | blockedOnce(t) |
| 303 | asked := 0 |
| 304 | lost := heldReleaseRelayFunc(func(ctx context.Context, q heldReleaseRequest) (heldReleaseResponse, error) { |
| 305 | asked++ |
| 306 | if _, err := root.Serve(ctx, hrPeer, q); err != nil { |
| 307 | return heldReleaseResponse{}, err |
| 308 | } |
| 309 | return heldReleaseResponse{}, errors.New("the connection dropped before the answer") |
| 310 | }) |
| 311 | ready, _ := heldReleaseHooksVia(heldReleaseVia{Relay: lost, Region: "uk"}, claimWrites(&heldSelections{}), testRegistry(t), opt, data, "uk-box", |
| 312 | hsPub, hsRatchet(t), offProofGate(t), nil) |
| 313 | if c := ready(ctx, h.s, rec, "v7"); c.OK { |
| 314 | t.Fatalf("T7 passed on a lost answer: %+v", c) |
| 315 | } |
| 316 | if c := ready(ctx, h.s, rec, "v7"); !c.OK { |
| 317 | t.Fatalf("T7 after the lost answer: %+v", c) |
| 318 | } |
| 319 | if asked != 1 { |
| 320 | t.Fatalf("root asked %d times, want once: the staged bytes are verified by the agent itself", asked) |
| 321 | } |
| 322 | }) |
| 323 | } |
| 324 | |
| 325 | // TestHeldReleaseRecoveryProbesFailClosed (fix round 1, item 2; |
| 326 | // platform-1e's ruling): a clean return's probe and a revert's probe never |
| 327 | // run configured, so nothing journals a render hash under their hold op. |
| 328 | // With the recovery release missing from live/<host>, each fails closed: the |
| 329 | // probe's release adapter gets a typed not-yet and asks root nothing, root |
| 330 | // asked directly refuses the row outright (fix round 2, P2-1), and nothing |
| 331 | // is published. A known limitation: such a probe would need to journal its own |
| 332 | // render hash through RenderHeldUnderBefore at an equivalent point. |
| 333 | func TestHeldReleaseRecoveryProbesFailClosed(t *testing.T) { |
| 334 | ctx := context.Background() |
| 335 | recovery := func(f *hrFixture) { |
| 336 | f.claim.Op.Exec, f.claim.Op.Participants = "uk-box", []string{"uk-box", "ie-box"} |
| 337 | f.claim.Op.Intent = map[string]string{"source": "uk-box", "target": "ie-box", "to": "ie"} |
| 338 | f.claim.Op.Phase = hostclaim.PhaseProbing |
| 339 | f.rec.Move.From, f.rec.Move.To, f.rec.Move.LandingPhase = "uk", "ie", "" |
| 340 | } |
| 341 | for _, tc := range []struct { |
| 342 | name string |
| 343 | set func(f *hrFixture) |
| 344 | hold string |
| 345 | }{ |
| 346 | {"a clean return's probe", func(f *hrFixture) { |
| 347 | recovery(f) |
| 348 | f.claim.Op.Kind = hostclaim.KindRestoring |
| 349 | f.claim.Op.Intent["rollback"] = hrRB |
| 350 | f.rec.Move.State = provision.MoveRollingBack |
| 351 | f.rec.Move.Rollback = &provision.MoveRollback{ID: hrRB, SourcePhase: provision.RbSourceProbing, RecoveryInputs: &provision.MoveInputs{Release: "v7"}} |
| 352 | f.row.HoldOp = hrRB |
| 353 | }, hrRB}, |
| 354 | {"a revert's probe", func(f *hrFixture) { |
| 355 | recovery(f) |
| 356 | f.claim.Op.Kind = hostclaim.KindReverting |
| 357 | f.claim.Op.Journal = map[string]string{hostclaim.JournalRevPhase: hostclaim.PhaseProbing} |
| 358 | f.rec.Move.State = provision.MoveReverting |
| 359 | f.rec.Move.RecoveryInputs = &provision.MoveInputs{Release: "v7"} |
| 360 | }, p1ID}, |
| 361 | } { |
| 362 | t.Run(tc.name, func(t *testing.T) { |
| 363 | f := newHRFixture(t) |
| 364 | f.set(tc.set) |
| 365 | // The recovery never ran configured: no render journal at all. |
| 366 | f.data = t.TempDir() |
| 367 | f.root.DataDir = f.data |
| 368 | var seen []heldReleaseRequest |
| 369 | adapter := heldReleaseProbeRelease(heldReleaseVia{Relay: f.relay(&seen), Region: "uk", Claims: f.claims}, f.s, "uk-box", f.opt, f.data, offProofGate(t)) |
| 370 | rt := registry.Route{Host: p1Host, Kind: "instance", App: "app", Account: "bab", HoldOp: tc.hold} |
| 371 | if err := adapter(ctx, rt, "v7", heldSelectionWork{&heldSelections{}}); !errors.Is(err, errHeldReleaseNotYet) { |
| 372 | t.Fatalf("the probe's release: %v, want errHeldReleaseNotYet", err) |
| 373 | } |
| 374 | if len(seen) != 0 { |
| 375 | t.Fatalf("root was asked %+v", seen) |
| 376 | } |
| 377 | // Root asked directly, as a request carrying a render hash would: |
| 378 | // since fix round 2 (P2-1) root refuses these rows outright, |
| 379 | // before any journal read (TestHeldReleaseRootRefusesUnjournalledRecoveryProbes |
| 380 | // covers a planted journal). |
| 381 | q := f.request(heldReleasePublish) |
| 382 | if q.HoldOp != tc.hold { |
| 383 | t.Fatalf("request hold op %q, want %q", q.HoldOp, tc.hold) |
| 384 | } |
| 385 | if _, err := f.root.Serve(ctx, hrPeer, q); !errors.Is(err, release.ErrStaleHeldStage) { |
| 386 | t.Fatalf("root: %v, want a refusal", err) |
| 387 | } |
| 388 | f.untouched(t) |
| 389 | }) |
| 390 | } |
| 391 | } |
| 392 | |
amadan