| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "log" |
| 9 | "os" |
| 10 | "path/filepath" |
| 11 | "strings" |
| 12 | "sync" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/hostclaim" |
| 15 | "github.com/carlosframework/platform/internal/release" |
| 16 | ) |
| 17 | |
| 18 | // relocationOpJournals are the files under relocate/ops/<op>/ that name |
| 19 | // their host inside: the install and attempt journals and the probe stop's |
| 20 | // fence owner. |
| 21 | var relocationOpJournals = []string{"install.json", "attempts.json", "probe-fence.json"} |
| 22 | |
| 23 | // relocationStraySaid throttles the sweep's line about an entry under |
| 24 | // relocate/ that is no relocation journal to once per path per process. |
| 25 | var relocationStraySaid sync.Map |
| 26 | |
| 27 | func sayRelocationStray(path string) { |
| 28 | if _, said := relocationStraySaid.LoadOrStore(path, true); !said { |
| 29 | log.Printf("relocation sweep: %s is not a relocation journal; left in place for an operator", path) |
| 30 | } |
| 31 | } |
| 32 | |
| 33 | // sweepRelocationJournals is the lifecycle half of N9 (ruling 61). Every |
| 34 | // region HA journal under <DataDir>/relocate names its op in its path and |
| 35 | // its host inside (the build scratch names its host in its file name): the |
| 36 | // install, attempt and probe-fence journals under ops/<op>/, the release |
| 37 | // positions at release/<op>.json and the build's scratch under build/<op>/. |
| 38 | // Each is removed once the host's claim no longer runs that op: replaced |
| 39 | // by a successor, the claim gone, or completed and marked discharged (a |
| 40 | // completed op's journals stay until that mark, ruling 76). While the op |
| 41 | // runs, every one of them stays. |
| 42 | // |
| 43 | // A journal that cannot be read is left in place and reported, and so is |
| 44 | // a claim whose completed relocation cannot be read: both are |
| 45 | // census-opaque, so they block nothing, and absence is never evidence. |
| 46 | // What no committed journal can be is removed: an ops directory holding |
| 47 | // no host-bearing journal, an atomic write's leftover temporary under |
| 48 | // release/, a build directory left empty. An entry the sweep does not know |
| 49 | // is left in place and logged once. |
| 50 | // |
| 51 | // It runs in two halves (Task 10 review M6). The first reads the |
| 52 | // directories and the claims with no lock held, and decides. The second |
| 53 | // holds relocationJournals exclusively (carry 3), so it never runs beside |
| 54 | // a relocation journal write in this process (the pass's own steps write |
| 55 | // in turn, the wake's probe attempts write attempts.json whenever a probe |
| 56 | // runs), and removes only what it finds, read again under the lock, as it |
| 57 | // was decided: the same host's journals for an op that is over, or still |
| 58 | // no committed journal at all. A claim's "the op is over" never reverts |
| 59 | // (op ids are never reused), so a decision read before the lock stands; |
| 60 | // the local files are what a write can change, and they are read again. |
| 61 | // |
| 62 | // The build's scratch (relocationbuild.go, MkdirAll then BuildGeneration) |
| 63 | // is written outside the lock, by Participant.Step on the pass's own |
| 64 | // goroutine, so it is serialized with the sweep by program order. A build |
| 65 | // moved off that goroutine would need the lock too. |
| 66 | func sweepRelocationJournals(ctx context.Context, dataDir string, claims release.Store) error { |
| 67 | root := filepath.Join(dataDir, "relocate") |
| 68 | var errs []error |
| 69 | gone := func(host, op string) (bool, error) { |
| 70 | obs, err := hostclaim.Read(ctx, claims, host) |
| 71 | if err != nil { |
| 72 | return false, err |
| 73 | } |
| 74 | if obs.Absent || obs.Claim == nil { |
| 75 | return true, nil |
| 76 | } |
| 77 | // Ruling 76: a completed relocation's journals, the probe stop's |
| 78 | // fence owner among them, stay until the box it completed on has |
| 79 | // discharged, released its hold, lifted its fence and marked it. |
| 80 | // A record that cannot be read keeps every journal: it may be this |
| 81 | // op's, and absence is never evidence. |
| 82 | done, ok, err := hostclaim.ReadCompletedRelocation(obs.Claim) |
| 83 | if err != nil { |
| 84 | return false, fmt.Errorf("%s: %w", host, err) |
| 85 | } |
| 86 | if ok && done.Op == op && !done.Discharged { |
| 87 | return false, nil |
| 88 | } |
| 89 | return obs.Claim.Op.ID != op, nil |
| 90 | } |
| 91 | // A removal decided unlocked: check reads the entry again under the |
| 92 | // lock and answers whether it is still as decided. |
| 93 | type removal struct { |
| 94 | path string |
| 95 | all bool // RemoveAll, not Remove |
| 96 | check func() bool |
| 97 | } |
| 98 | var removals []removal |
| 99 | // ops/<op>/{install,attempts,probe-fence}.json |
| 100 | ops, err := os.ReadDir(filepath.Join(root, "ops")) |
| 101 | if err != nil && !errors.Is(err, os.ErrNotExist) { |
| 102 | return err |
| 103 | } |
| 104 | for _, e := range ops { |
| 105 | dir := filepath.Join(root, "ops", e.Name()) |
| 106 | if !e.IsDir() { |
| 107 | sayRelocationStray(dir) |
| 108 | continue |
| 109 | } |
| 110 | host, found, readErr := opJournalHost(dir) |
| 111 | switch { |
| 112 | case host != "": |
| 113 | if done, err := gone(host, e.Name()); err != nil { |
| 114 | errs = append(errs, err) |
| 115 | } else if done { |
| 116 | removals = append(removals, removal{path: dir, all: true, check: func() bool { |
| 117 | h, _, err := opJournalHost(dir) |
| 118 | return err == nil && (h == host || h == "") |
| 119 | }}) |
| 120 | } |
| 121 | case found: |
| 122 | errs = append(errs, readErr) // unreadable: kept and reported |
| 123 | default: |
| 124 | // No host-bearing journal was ever committed here: what is |
| 125 | // left is a write's temporary, which no op can need, unless a |
| 126 | // write committed one since (read again under the lock). |
| 127 | removals = append(removals, removal{path: dir, all: true, check: func() bool { |
| 128 | _, found, _ := opJournalHost(dir) |
| 129 | return !found |
| 130 | }}) |
| 131 | } |
| 132 | } |
| 133 | // release/<op>.json, and an atomic write's leftover temporaries. |
| 134 | positions, err := os.ReadDir(filepath.Join(root, "release")) |
| 135 | if err != nil && !errors.Is(err, os.ErrNotExist) { |
| 136 | return errors.Join(append(errs, err)...) |
| 137 | } |
| 138 | for _, e := range positions { |
| 139 | path := filepath.Join(root, "release", e.Name()) |
| 140 | op, isJSON := strings.CutSuffix(e.Name(), ".json") |
| 141 | switch { |
| 142 | case e.IsDir(): |
| 143 | sayRelocationStray(path) |
| 144 | case isJSON: |
| 145 | host, err := relocationJournalHost(path) |
| 146 | if err != nil { |
| 147 | errs = append(errs, err) |
| 148 | continue |
| 149 | } |
| 150 | if done, err := gone(host, op); err != nil { |
| 151 | errs = append(errs, err) |
| 152 | } else if done { |
| 153 | removals = append(removals, removal{path: path, check: func() bool { |
| 154 | h, err := relocationJournalHost(path) |
| 155 | return err == nil && h == host |
| 156 | }}) |
| 157 | } |
| 158 | case relocationWriteLeftover(e.Name()): |
| 159 | // Under the lock no write is in flight, so a temporary still |
| 160 | // there is a leftover, never a write's. |
| 161 | removals = append(removals, removal{path: path, check: func() bool { return true }}) |
| 162 | default: |
| 163 | sayRelocationStray(path) |
| 164 | } |
| 165 | } |
| 166 | // build/<op>/<host>.db and its SQLite sidecars: the scratch of a build |
| 167 | // whose op is over. |
| 168 | builds, err := os.ReadDir(filepath.Join(root, "build")) |
| 169 | if err != nil && !errors.Is(err, os.ErrNotExist) { |
| 170 | return errors.Join(append(errs, err)...) |
| 171 | } |
| 172 | for _, e := range builds { |
| 173 | dir := filepath.Join(root, "build", e.Name()) |
| 174 | if !e.IsDir() { |
| 175 | sayRelocationStray(dir) |
| 176 | continue |
| 177 | } |
| 178 | host, empty, err := buildScratchDirHost(dir) |
| 179 | switch { |
| 180 | case err != nil: |
| 181 | errs = append(errs, err) |
| 182 | case host != "": |
| 183 | if done, err := gone(host, e.Name()); err != nil { |
| 184 | errs = append(errs, err) |
| 185 | } else if done { |
| 186 | removals = append(removals, removal{path: dir, all: true, check: func() bool { |
| 187 | h, _, err := buildScratchDirHost(dir) |
| 188 | return err == nil && (h == host || h == "") |
| 189 | }}) |
| 190 | } |
| 191 | case empty: |
| 192 | removals = append(removals, removal{path: dir, check: func() bool { |
| 193 | _, empty, err := buildScratchDirHost(dir) |
| 194 | return err == nil && empty |
| 195 | }}) |
| 196 | default: |
| 197 | sayRelocationStray(dir) |
| 198 | } |
| 199 | } |
| 200 | if len(removals) == 0 { |
| 201 | return errors.Join(errs...) |
| 202 | } |
| 203 | relocationJournals.Lock() |
| 204 | defer relocationJournals.Unlock() |
| 205 | for _, r := range removals { |
| 206 | if _, err := os.Lstat(r.path); errors.Is(err, os.ErrNotExist) { |
| 207 | continue |
| 208 | } |
| 209 | if !r.check() { |
| 210 | continue // a write landed since the decision: the next pass decides again |
| 211 | } |
| 212 | if r.all { |
| 213 | errs = append(errs, os.RemoveAll(r.path)) |
| 214 | } else { |
| 215 | errs = append(errs, os.Remove(r.path)) |
| 216 | } |
| 217 | } |
| 218 | return errors.Join(errs...) |
| 219 | } |
| 220 | |
| 221 | // relocationJournalHost is the host a relocation journal names inside: |
| 222 | // "host", or a stop request's "Request.Host" (release positions). |
| 223 | func relocationJournalHost(path string) (string, error) { |
| 224 | raw, err := os.ReadFile(path) |
| 225 | if err != nil { |
| 226 | return "", err |
| 227 | } |
| 228 | var probe struct { |
| 229 | Host string `json:"host"` |
| 230 | Request struct{ Host string } |
| 231 | } |
| 232 | if err := json.Unmarshal(raw, &probe); err != nil { |
| 233 | return "", fmt.Errorf("%s: %w", path, err) |
| 234 | } |
| 235 | if probe.Host != "" { |
| 236 | return probe.Host, nil |
| 237 | } |
| 238 | if probe.Request.Host != "" { |
| 239 | return probe.Request.Host, nil |
| 240 | } |
| 241 | return "", fmt.Errorf("%s names no host", path) |
| 242 | } |
| 243 | |
| 244 | // opJournalHost is the host an ops/<op> directory's journals name: the |
| 245 | // first readable one, found when any journal is present at all, and the |
| 246 | // read errors of the unreadable ones when none is readable. |
| 247 | func opJournalHost(dir string) (host string, found bool, err error) { |
| 248 | for _, name := range relocationOpJournals { |
| 249 | h, rerr := relocationJournalHost(filepath.Join(dir, name)) |
| 250 | if errors.Is(rerr, os.ErrNotExist) { |
| 251 | continue |
| 252 | } |
| 253 | found = true |
| 254 | if rerr == nil { |
| 255 | return h, true, nil |
| 256 | } |
| 257 | err = errors.Join(err, rerr) |
| 258 | } |
| 259 | return "", found, err |
| 260 | } |
| 261 | |
| 262 | // buildScratchDirHost is the host a build/<op> directory's scratch names, |
| 263 | // and whether the directory is empty. |
| 264 | func buildScratchDirHost(dir string) (string, bool, error) { |
| 265 | files, err := os.ReadDir(dir) |
| 266 | if err != nil { |
| 267 | return "", false, err |
| 268 | } |
| 269 | for _, f := range files { |
| 270 | if h, ok := buildScratchHost(f.Name()); ok { |
| 271 | return h, false, nil |
| 272 | } |
| 273 | } |
| 274 | return "", len(files) == 0, nil |
| 275 | } |
| 276 | |
| 277 | // relocationJournalHosts is every host this box's op journals name |
| 278 | // (ops/*/{probe-fence,attempts}.json): the completion-only pass's positive |
| 279 | // local evidence of a relocation this box took part in, written before any |
| 280 | // completion exists and kept until its mark (ruling 76; Task 10 review |
| 281 | // M1). A journal that cannot be read is returned as an error beside the |
| 282 | // hosts that could. |
| 283 | func relocationJournalHosts(dataDir string) ([]string, error) { |
| 284 | ops, err := os.ReadDir(filepath.Join(dataDir, "relocate", "ops")) |
| 285 | if errors.Is(err, os.ErrNotExist) { |
| 286 | return nil, nil |
| 287 | } |
| 288 | if err != nil { |
| 289 | return nil, err |
| 290 | } |
| 291 | seen := map[string]bool{} |
| 292 | var out []string |
| 293 | var errs []error |
| 294 | for _, e := range ops { |
| 295 | if !e.IsDir() { |
| 296 | continue |
| 297 | } |
| 298 | for _, name := range []string{"probe-fence.json", "attempts.json"} { |
| 299 | host, err := relocationJournalHost(filepath.Join(dataDir, "relocate", "ops", e.Name(), name)) |
| 300 | if errors.Is(err, os.ErrNotExist) { |
| 301 | continue |
| 302 | } |
| 303 | if err != nil { |
| 304 | errs = append(errs, err) |
| 305 | continue |
| 306 | } |
| 307 | if !seen[host] { |
| 308 | seen[host] = true |
| 309 | out = append(out, host) |
| 310 | } |
| 311 | } |
| 312 | } |
| 313 | return out, errors.Join(errs...) |
| 314 | } |
| 315 | |
| 316 | // relocationJournals serializes the journal sweep against every relocation |
| 317 | // journal write in this process (carry 3). A writer holds it shared from |
| 318 | // creating the journal's directory until its atomic rename has landed; the |
| 319 | // sweep holds it exclusively. Without it the sweep could find an ops |
| 320 | // directory between the writer's MkdirAll and its rename, holding no |
| 321 | // committed journal, and remove it under the write. |
| 322 | var relocationJournals sync.RWMutex |
| 323 | |
| 324 | // writeRelocationJournal is every relocation journal write under |
| 325 | // <DataDir>/relocate: the directory, then moves' writeFileAtomic, under |
| 326 | // relocationJournals shared. |
| 327 | func writeRelocationJournal(path string, raw []byte) error { |
| 328 | relocationJournals.RLock() |
| 329 | defer relocationJournals.RUnlock() |
| 330 | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { |
| 331 | return err |
| 332 | } |
| 333 | return writeFileAtomic(path, raw, 0o600) |
| 334 | } |
| 335 | |
| 336 | // relocationWriteLeftover is a temporary an interrupted atomic write left |
| 337 | // under release/: writeFileAtomic's ".<op>.json.<random>", or a fixed |
| 338 | // "<op>.json.tmp". |
| 339 | func relocationWriteLeftover(name string) bool { |
| 340 | return strings.HasSuffix(name, ".json.tmp") || (strings.HasPrefix(name, ".") && strings.Contains(name, ".json.")) |
| 341 | } |
| 342 | |
| 343 | // buildScratchHost is the host a build scratch file names: <host>.db, or |
| 344 | // one of SQLite's sidecars beside it (-wal, -shm, -journal). |
| 345 | func buildScratchHost(name string) (string, bool) { |
| 346 | for _, sidecar := range []string{"-wal", "-shm", "-journal"} { |
| 347 | if base, ok := strings.CutSuffix(name, sidecar); ok { |
| 348 | name = base |
| 349 | break |
| 350 | } |
| 351 | } |
| 352 | host, ok := strings.CutSuffix(name, ".db") |
| 353 | return host, ok && host != "" |
| 354 | } |
| 355 | |
amadan