| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "database/sql" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "net/http" |
| 9 | "net/http/httptest" |
| 10 | "path/filepath" |
| 11 | "sync/atomic" |
| 12 | "testing" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/hostbox" |
| 15 | "github.com/carlosframework/platform/internal/hostguard" |
| 16 | _ "modernc.org/sqlite" |
| 17 | ) |
| 18 | |
| 19 | // A receipt lookup that fails says nothing about the request. A replay of the |
| 20 | // retained request after a lost reply must never become a definite refusal. |
| 21 | func TestPlan9JournalReplayLookupFailureIsUncertain(t *testing.T) { |
| 22 | path := filepath.Join(t.TempDir(), "root.db") |
| 23 | j, err := hostbox.OpenLaunchJournal(path) |
| 24 | if err != nil { |
| 25 | t.Fatal(err) |
| 26 | } |
| 27 | defer j.Close() |
| 28 | s, err := hostbox.NewJournalService(j) |
| 29 | if err != nil { |
| 30 | t.Fatal(err) |
| 31 | } |
| 32 | in := hostbox.LaunchInventory{Target: hostguard.FenceTarget{Host: "app.example", Incarnation: "inc", Stamp: 1, Session: 1, Box: "box", Enrollment: "enroll", BootID: "boot", ControllerID: "controller"}, ControllerPID: 17, ProcessStart: "23", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "session", Device: 1, Inode: 2}}} |
| 33 | q := plan9CanonicalJournalRequest(t, hostbox.JournalRequest{V: 1, Op: "create", Launch: in}) |
| 34 | if _, err = s.Execute(t.Context(), "box/enroll", q); err != nil { |
| 35 | t.Fatal(err) |
| 36 | } |
| 37 | root := ClaimRootHTTP{Journals: s} |
| 38 | root.Reconcile.Box = "box" |
| 39 | root.Reconcile.Enrollment = "enroll" |
| 40 | peer := StopPeer{UID: 1001, PID: 17, ProcessStart: "23", BootID: "boot", ControllerID: "controller"} |
| 41 | // The client resends the byte-identical request and ID it retained. |
| 42 | replay := func() *httptest.ResponseRecorder { |
| 43 | body, _ := json.Marshal(q) |
| 44 | response := httptest.NewRecorder() |
| 45 | root.serveJournal(response, httptest.NewRequest(http.MethodPost, "/claim/journal", bytes.NewReader(body)), peer) |
| 46 | return response |
| 47 | } |
| 48 | // Break only the receipt lookup, from a second connection. |
| 49 | other, err := sql.Open("sqlite", "file:"+path+"?_pragma=busy_timeout(5000)") |
| 50 | if err != nil { |
| 51 | t.Fatal(err) |
| 52 | } |
| 53 | defer other.Close() |
| 54 | if _, err = other.Exec("ALTER TABLE journal_requests RENAME TO journal_requests_hidden"); err != nil { |
| 55 | t.Fatal(err) |
| 56 | } |
| 57 | // 400/403/409 are definite refusals to the client; only 5xx stays uncertain. |
| 58 | if response := replay(); response.Code < 500 { |
| 59 | t.Fatalf("failed receipt lookup reported as definite result %d: %s", response.Code, response.Body.String()) |
| 60 | } |
| 61 | if _, err = other.Exec("ALTER TABLE journal_requests_hidden RENAME TO journal_requests"); err != nil { |
| 62 | t.Fatal(err) |
| 63 | } |
| 64 | var out hostbox.JournalResponse |
| 65 | response := replay() |
| 66 | if err = json.Unmarshal(response.Body.Bytes(), &out); response.Code != http.StatusOK || err != nil || out.Error != "" || out.ID != q.ID { |
| 67 | t.Fatalf("replay did not recover once lookup worked: %d %+v %v", response.Code, out, err) |
| 68 | } |
| 69 | var receipts int |
| 70 | if err = other.QueryRow("SELECT COUNT(*) FROM journal_requests").Scan(&receipts); err != nil || receipts != 1 { |
| 71 | t.Fatalf("duplicate effect or receipt: %d %v", receipts, err) |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | // A byte-different request under a stored ID is a proven conflict. |
| 76 | func TestPlan9JournalReplayMismatchIsConflict(t *testing.T) { |
| 77 | j, err := hostbox.OpenLaunchJournal(filepath.Join(t.TempDir(), "root.db")) |
| 78 | if err != nil { |
| 79 | t.Fatal(err) |
| 80 | } |
| 81 | defer j.Close() |
| 82 | s, err := hostbox.NewJournalService(j) |
| 83 | if err != nil { |
| 84 | t.Fatal(err) |
| 85 | } |
| 86 | in := hostbox.LaunchInventory{Target: hostguard.FenceTarget{Host: "app.example", Incarnation: "inc", Stamp: 1, Session: 1, Box: "box", Enrollment: "enroll", BootID: "boot", ControllerID: "controller"}, ControllerPID: 17, ProcessStart: "23", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "session", Device: 1, Inode: 2}}} |
| 87 | q := plan9CanonicalJournalRequest(t, hostbox.JournalRequest{V: 1, Op: "create", Launch: in}) |
| 88 | if _, err = s.Execute(t.Context(), "box/enroll", q); err != nil { |
| 89 | t.Fatal(err) |
| 90 | } |
| 91 | if _, _, err = s.Replay(t.Context(), "other/peer", q); !errors.Is(err, hostbox.ErrChanged) { |
| 92 | t.Fatalf("different owner not a proven mismatch: %v", err) |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | // A request committed and its reply is lost. A new rootJournalClient replays |
| 97 | // the retained request. While the receipt lookup fails it must report |
| 98 | // JournalUncertainError, never a definite refusal; once the lookup works the |
| 99 | // same request replays without a second effect. |
| 100 | func TestPlan9JournalRestartedClientLostReplyLookupFailure(t *testing.T) { |
| 101 | path := filepath.Join(t.TempDir(), "root.db") |
| 102 | j, err := hostbox.OpenLaunchJournal(path) |
| 103 | if err != nil { |
| 104 | t.Fatal(err) |
| 105 | } |
| 106 | defer j.Close() |
| 107 | s, err := hostbox.NewJournalService(j) |
| 108 | if err != nil { |
| 109 | t.Fatal(err) |
| 110 | } |
| 111 | root := ClaimRootHTTP{Journals: s} |
| 112 | root.Reconcile.Box = "box" |
| 113 | root.Reconcile.Enrollment = "enroll" |
| 114 | peer := StopPeer{UID: 1001, PID: 17, ProcessStart: "23", BootID: "boot", ControllerID: "controller"} |
| 115 | in := hostbox.LaunchInventory{Target: hostguard.FenceTarget{Host: "app.example", Incarnation: "inc", Stamp: 1, Session: 1, Box: "box", Enrollment: "enroll", BootID: "boot", ControllerID: "controller"}, ControllerPID: 17, ProcessStart: "23", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "session", Device: 1, Inode: 2}}} |
| 116 | if err = j.Create(t.Context(), in); err != nil { |
| 117 | t.Fatal(err) |
| 118 | } |
| 119 | if err = j.RecordProducer(t.Context(), in.Target, "issued"); err != nil { |
| 120 | t.Fatal(err) |
| 121 | } |
| 122 | var loseReply atomic.Bool |
| 123 | var requests atomic.Int32 |
| 124 | handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 125 | requests.Add(1) |
| 126 | if !loseReply.Load() { |
| 127 | root.serveJournal(w, r, peer) |
| 128 | return |
| 129 | } |
| 130 | // Commit, then drop the connection before any reply is written. |
| 131 | root.serveJournal(httptest.NewRecorder(), r, peer) |
| 132 | if conn, _, err := w.(http.Hijacker).Hijack(); err == nil { |
| 133 | conn.Close() |
| 134 | } |
| 135 | }) |
| 136 | countReceipts := func(db *sql.DB) int { |
| 137 | var n int |
| 138 | if err := db.QueryRow("SELECT COUNT(*) FROM journal_requests").Scan(&n); err != nil { |
| 139 | t.Fatal(err) |
| 140 | } |
| 141 | return n |
| 142 | } |
| 143 | other, err := sql.Open("sqlite", "file:"+path+"?_pragma=busy_timeout(5000)") |
| 144 | if err != nil { |
| 145 | t.Fatal(err) |
| 146 | } |
| 147 | defer other.Close() |
| 148 | |
| 149 | // First client: the request commits and every reply is lost. |
| 150 | loseReply.Store(true) |
| 151 | first, _ := hostbox.RemoteJournals(plan9JournalSocket(t, handler)) |
| 152 | var uncertain *hostbox.JournalUncertainError |
| 153 | if _, err = first.CompleteProducer(t.Context(), in.Target, "issued", hostbox.ProducerCompletion{}); !errors.As(err, &uncertain) { |
| 154 | t.Fatalf("lost reply was not uncertain: %v", err) |
| 155 | } |
| 156 | if n := countReceipts(other); n != 1 { |
| 157 | t.Fatalf("request did not commit exactly once: %d", n) |
| 158 | } |
| 159 | |
| 160 | // Restarted client: receipt lookup fails on the server. |
| 161 | loseReply.Store(false) |
| 162 | if _, err = other.Exec("ALTER TABLE journal_requests RENAME TO journal_requests_hidden"); err != nil { |
| 163 | t.Fatal(err) |
| 164 | } |
| 165 | before := requests.Load() |
| 166 | restarted, _ := hostbox.RemoteJournals(plan9JournalSocket(t, handler)) |
| 167 | retained := uncertain |
| 168 | _, err = restarted.ReplayUncertain(t.Context(), retained) |
| 169 | if !errors.As(err, &uncertain) { |
| 170 | t.Fatalf("failed lookup was not JournalUncertainError (definite refusal?): %v", err) |
| 171 | } |
| 172 | if requests.Load()-before != 2 { |
| 173 | t.Fatalf("restarted client attempts = %d, want 2", requests.Load()-before) |
| 174 | } |
| 175 | |
| 176 | // Lookup restored: another fresh client replays the stored result. |
| 177 | if _, err = other.Exec("ALTER TABLE journal_requests_hidden RENAME TO journal_requests"); err != nil { |
| 178 | t.Fatal(err) |
| 179 | } |
| 180 | again, _ := hostbox.RemoteJournals(plan9JournalSocket(t, handler)) |
| 181 | if _, err = again.ReplayUncertain(t.Context(), retained); err != nil { |
| 182 | t.Fatalf("replay after restore failed: %v", err) |
| 183 | } |
| 184 | if n := countReceipts(other); n != 1 { |
| 185 | t.Fatalf("duplicate effect or receipt: %d", n) |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | // HTTP classification of Replay errors: a proven mismatch is 409, an invalid |
| 190 | // request is 400, and only ErrChanged maps to 409. |
| 191 | func TestPlan9JournalReplayErrorStatuses(t *testing.T) { |
| 192 | j, err := hostbox.OpenLaunchJournal(filepath.Join(t.TempDir(), "root.db")) |
| 193 | if err != nil { |
| 194 | t.Fatal(err) |
| 195 | } |
| 196 | defer j.Close() |
| 197 | s, err := hostbox.NewJournalService(j) |
| 198 | if err != nil { |
| 199 | t.Fatal(err) |
| 200 | } |
| 201 | in := hostbox.LaunchInventory{Target: hostguard.FenceTarget{Host: "app.example", Incarnation: "inc", Stamp: 1, Session: 1, Box: "box", Enrollment: "enroll", BootID: "boot", ControllerID: "controller"}, ControllerPID: 17, ProcessStart: "23", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "session", Device: 1, Inode: 2}}} |
| 202 | q := plan9CanonicalJournalRequest(t, hostbox.JournalRequest{V: 1, Op: "create", Launch: in}) |
| 203 | if _, err = s.Execute(t.Context(), "box/enroll", q); err != nil { |
| 204 | t.Fatal(err) |
| 205 | } |
| 206 | peer := StopPeer{UID: 1001, PID: 17, ProcessStart: "23", BootID: "boot", ControllerID: "controller"} |
| 207 | serve := func(box string, q hostbox.JournalRequest) int { |
| 208 | root := ClaimRootHTTP{Journals: s} |
| 209 | root.Reconcile.Box = box |
| 210 | root.Reconcile.Enrollment = "enroll" |
| 211 | body, _ := json.Marshal(q) |
| 212 | response := httptest.NewRecorder() |
| 213 | root.serveJournal(response, httptest.NewRequest(http.MethodPost, "/claim/journal", bytes.NewReader(body)), peer) |
| 214 | return response.Code |
| 215 | } |
| 216 | if got := serve("other", q); got != http.StatusConflict { |
| 217 | t.Fatalf("other owner under stored ID = %d, want 409", got) |
| 218 | } |
| 219 | tampered := q |
| 220 | tampered.Launch.ControllerPID++ |
| 221 | if got := serve("box", tampered); got != http.StatusConflict { |
| 222 | t.Fatalf("changed bytes under stored ID = %d, want 409", got) |
| 223 | } |
| 224 | invalid := q |
| 225 | invalid.V = 0 |
| 226 | if got := serve("box", invalid); got != http.StatusBadRequest { |
| 227 | t.Fatalf("invalid request = %d, want 400", got) |
| 228 | } |
| 229 | } |
| 230 | |
amadan