| 1 | #!/bin/sh |
| 2 | # The test half of `make test`, and so of CI: `.amadan/ci.d/10-gate` runs |
| 3 | # `make test`, which runs this. Still one definition. |
| 4 | # |
| 5 | # What it checks is what `make test` always checked: launchctx's vet and |
| 6 | # tests, `go vet ./...`, a repo-wide `gofmt -l .`, and every test in every |
| 7 | # package with -count=1. What changed is the shape of the run. |
| 8 | # |
| 9 | # The gate's wall clock was the slowest single package, because Go runs |
| 10 | # packages side by side but a package's own tests one after another. |
| 11 | # internal/console took about 440 s on CI's 32 cores while most of them |
| 12 | # sat idle, and the root package was not far behind. So: |
| 13 | # |
| 14 | # - vet, gofmt, launchctx and the tests all start at once instead of |
| 15 | # vet finishing before the first test starts; |
| 16 | # - the packages named in TEST_SPLIT have their top-level tests dealt |
| 17 | # across several `go test -run` processes running concurrently. |
| 18 | # |
| 19 | # The deal is by measured weight (.amadan/test-weights.tsv, longest first |
| 20 | # onto the least-loaded shard), so one slow test gets a shard to itself |
| 21 | # rather than dragging its neighbours along. A test missing from the |
| 22 | # weights file still runs; it is only dealt as if it were short. Stale |
| 23 | # weights cost balance, never coverage. |
| 24 | # |
| 25 | # THE FENCE. Each split package's test list comes from `go test -list`, |
| 26 | # every name on it goes to exactly one shard, and a shard whose -run |
| 27 | # pattern matches nothing fails the gate. A test can therefore be added, |
| 28 | # renamed or removed without touching this file, and no edit to the |
| 29 | # deal can drop one silently. |
| 30 | # |
| 31 | # Shards are separate processes, so tests that leaned on running in the |
| 32 | # same process as some other test (package state one test leaves for the |
| 33 | # next) can surface here. That is a bug in the test, not in the split. |
| 34 | set -eu |
| 35 | |
| 36 | : "${TEST_SPLIT=./internal/console=8 ./internal/host=4 .=5}" |
| 37 | weights=$(pwd)/.amadan/test-weights.tsv |
| 38 | |
| 39 | # A shard count that is not a whole number of at least 1 would take a |
| 40 | # package out of the ordinary run and then launch none of its shards: a |
| 41 | # green gate that ran none of that package's tests. Refuse it up front. |
| 42 | for spec in $TEST_SPLIT; do |
| 43 | count=${spec#*=} |
| 44 | case $spec:$count in |
| 45 | *=*:[0-9]*) ;; |
| 46 | *) |
| 47 | echo "TEST_SPLIT entry '$spec' is not dir=count" >&2 |
| 48 | exit 2 |
| 49 | ;; |
| 50 | esac |
| 51 | case $count in |
| 52 | *[!0-9]*) |
| 53 | echo "TEST_SPLIT entry '$spec' needs a whole number of shards" >&2 |
| 54 | exit 2 |
| 55 | ;; |
| 56 | esac |
| 57 | if [ "$count" -lt 1 ]; then |
| 58 | echo "TEST_SPLIT entry '$spec' needs at least one shard" >&2 |
| 59 | exit 2 |
| 60 | fi |
| 61 | done |
| 62 | |
| 63 | # Stopping the gate stops its jobs: the go test processes and the test |
| 64 | # binaries under them, not just the shells that started them. That covers |
| 65 | # an interrupt, a TERM from the runner, and an early exit while jobs are |
| 66 | # already running. Process groups (set -m) would be simpler, but dash |
| 67 | # turns job control off without a terminal, which is how CI runs it, so |
| 68 | # each job's tree is walked with pgrep -P instead, and the gate refuses |
| 69 | # to start without it rather than leave orphans when it is stopped. |
| 70 | if ! command -v pgrep >/dev/null 2>&1; then |
| 71 | echo "pgrep is required: without it a stopped gate cannot stop its jobs" >&2 |
| 72 | exit 2 |
| 73 | fi |
| 74 | out=$(mktemp -d "${TMPDIR:-/tmp}/carlos-gate-XXXXXX") |
| 75 | # started holds name:pid for every job started; discovery is the one |
| 76 | # foreground command in flight, if any. A job is only signalled while its |
| 77 | # .secs file (the last thing it writes) is absent, so a finished job's |
| 78 | # pid, which the system may since have handed to someone else's process |
| 79 | # on this shared machine, is never touched. |
| 80 | started="" |
| 81 | discovery="" |
| 82 | kill_tree() { |
| 83 | for child in $(pgrep -P "$1" 2>/dev/null); do |
| 84 | kill_tree "$child" |
| 85 | done |
| 86 | kill -TERM "$1" 2>/dev/null || true |
| 87 | } |
| 88 | stop_jobs() { |
| 89 | if [ -n "$discovery" ]; then |
| 90 | kill_tree "$discovery" |
| 91 | fi |
| 92 | for job in $started; do |
| 93 | if [ ! -e "$out/${job%:*}.secs" ]; then |
| 94 | kill_tree "${job##*:}" |
| 95 | fi |
| 96 | done |
| 97 | wait 2>/dev/null || true |
| 98 | rm -rf "$out" |
| 99 | } |
| 100 | trap stop_jobs EXIT |
| 101 | trap 'trap - EXIT; stop_jobs; exit 130' INT TERM |
| 102 | |
| 103 | names="" |
| 104 | # A failure found before any test runs (a package that will not list, a |
| 105 | # deal that lost a test) is reported as a job of its own, so the jobs |
| 106 | # already running still finish and print rather than being cut off. |
| 107 | fail_job() { |
| 108 | names="$names $1" |
| 109 | printf '%s\n' "$2" >>"$out/$1.log" |
| 110 | echo 1 >"$out/$1.rc" |
| 111 | echo 0 >"$out/$1.secs" |
| 112 | } |
| 113 | start() { |
| 114 | start_name=$1 |
| 115 | shift |
| 116 | names="$names $start_name" |
| 117 | ( |
| 118 | set +e |
| 119 | t0=$(date +%s) |
| 120 | "$@" >"$out/$start_name.log" 2>&1 |
| 121 | echo $? >"$out/$start_name.rc" |
| 122 | echo $(($(date +%s) - t0)) >"$out/$start_name.secs" |
| 123 | ) & |
| 124 | started="$started $start_name:$!" |
| 125 | } |
| 126 | |
| 127 | # Runs one discovery command in the background and waits for it, so an |
| 128 | # interrupt reaches the trap at once instead of after a compile. |
| 129 | discover() { |
| 130 | "$@" & |
| 131 | discovery=$! |
| 132 | discover_rc=0 |
| 133 | wait "$discovery" || discover_rc=$? |
| 134 | discovery="" |
| 135 | return "$discover_rc" |
| 136 | } |
| 137 | |
| 138 | start launchctx sh -c 'cd launchctx && go vet ./... && go test ./... -count=1' |
| 139 | start vet go vet ./... |
| 140 | start gofmt sh -c 'fmt_out=$(gofmt -l .); if [ -n "$fmt_out" ]; then echo "gofmt needed:"; echo "$fmt_out"; exit 1; fi; echo "gofmt -l .: clean"' |
| 141 | |
| 142 | # Everything not split runs as one ordinary `go test`. Packages are named |
| 143 | # by import path, so match TEST_SPLIT's directories through `go list`. |
| 144 | split_paths="" |
| 145 | for spec in $TEST_SPLIT; do |
| 146 | discover go list "${spec%=*}" >"$out/go-list" |
| 147 | split_paths="$split_paths $(cat "$out/go-list")" |
| 148 | done |
| 149 | # go list runs on its own, not in a pipeline: a pipeline's status is its |
| 150 | # last command's, and a go list that failed halfway would hand on a short |
| 151 | # package list with a zero status. Each go list goes through discover, and |
| 152 | # set -e stops the gate on its status. |
| 153 | discover go list ./... >"$out/go-list" |
| 154 | all=$(cat "$out/go-list") |
| 155 | if [ -z "$all" ]; then |
| 156 | echo "go list ./... found no packages" >&2 |
| 157 | exit 1 |
| 158 | fi |
| 159 | rest="" |
| 160 | for p in $all; do |
| 161 | case " $split_paths " in |
| 162 | *" $p "*) ;; |
| 163 | *) rest="$rest $p" ;; |
| 164 | esac |
| 165 | done |
| 166 | # With no packages left, go test would test the current directory instead. |
| 167 | if [ -n "$rest" ]; then |
| 168 | # shellcheck disable=SC2086 # one word per package, on purpose |
| 169 | start rest go test -count=1 -timeout 30m $rest |
| 170 | fi |
| 171 | |
| 172 | for spec in $TEST_SPLIT; do |
| 173 | dir=${spec%=*} |
| 174 | shards=${spec#*=} |
| 175 | discover go list "$dir" >"$out/go-list" |
| 176 | pkg=$(cat "$out/go-list") |
| 177 | tag=$(echo "$dir" | sed 's|^\./||; s|^\.$|root|; s|/|_|g') |
| 178 | |
| 179 | # -list compiles the test binary, which the shards below then reuse |
| 180 | # from the build cache. Benchmarks are listed but never run by a plain |
| 181 | # `go test`, so they are left out of the deal. |
| 182 | if ! discover go test -count=1 -list '.*' "$dir" >"$out/list-$tag.raw" 2>&1; then |
| 183 | cp "$out/list-$tag.raw" "$out/$tag.log" |
| 184 | fail_job "$tag" "could not list the tests in $pkg" |
| 185 | continue |
| 186 | fi |
| 187 | grep -E '^(Test|Example|Fuzz)' "$out/list-$tag.raw" | sort -u >"$out/list-$tag" || true |
| 188 | listed=$(wc -l <"$out/list-$tag") |
| 189 | if [ "$listed" -eq 0 ]; then |
| 190 | fail_job "$tag" "go test -list found no tests in $pkg, which is not believable" |
| 191 | continue |
| 192 | fi |
| 193 | |
| 194 | # Longest-processing-time first: heaviest test onto the least-loaded |
| 195 | # shard. Unweighted tests count as 0.05 s (see test-weights.sh). |
| 196 | awk -v pkg="$pkg" -v n="$shards" -F '\t' ' |
| 197 | FILENAME == ARGV[1] { if ($1 == pkg) w[$2] = $3; next } |
| 198 | { print (($0 in w) ? w[$0] : 0.05) "\t" $0 } |
| 199 | ' "$weights" "$out/list-$tag" | sort -t "$(printf '\t')" -k1,1gr -k2,2 | |
| 200 | awk -v n="$shards" -F '\t' ' |
| 201 | { |
| 202 | best = 0 |
| 203 | for (i = 1; i < n; i++) if (load[i] < load[best]) best = i |
| 204 | load[best] += $1 |
| 205 | print best "\t" $2 |
| 206 | } |
| 207 | ' >"$out/deal-$tag" |
| 208 | |
| 209 | dealt=$(wc -l <"$out/deal-$tag") |
| 210 | if [ "$dealt" -ne "$listed" ]; then |
| 211 | fail_job "$tag" "dealt $dealt of $listed tests in $pkg" |
| 212 | continue |
| 213 | fi |
| 214 | |
| 215 | i=0 |
| 216 | launched=0 |
| 217 | while [ "$i" -lt "$shards" ]; do |
| 218 | pattern=$(awk -v i="$i" -F '\t' '$1 == i { printf "%s%s", sep, $2; sep = "|" }' "$out/deal-$tag") |
| 219 | if [ -n "$pattern" ]; then |
| 220 | start "$tag@$i" go test -count=1 -timeout 30m -run "^($pattern)\$" "$dir" |
| 221 | launched=$((launched + $(awk -v i="$i" -F '\t' '$1 == i' "$out/deal-$tag" | wc -l))) |
| 222 | fi |
| 223 | i=$((i + 1)) |
| 224 | done |
| 225 | if [ "$launched" -ne "$listed" ]; then |
| 226 | fail_job "$tag-deal" "launched $launched of $listed tests in $pkg" |
| 227 | fi |
| 228 | done |
| 229 | |
| 230 | wait |
| 231 | started="" |
| 232 | |
| 233 | failed="" |
| 234 | for name in $names; do |
| 235 | # A job with no status file was killed before it could write one. |
| 236 | rc=$(cat "$out/$name.rc" 2>/dev/null || echo 1) |
| 237 | echo "=== $name ($(cat "$out/$name.secs" 2>/dev/null || echo '?') s)" |
| 238 | cat "$out/$name.log" 2>/dev/null || true |
| 239 | case $name in |
| 240 | *@*) |
| 241 | if grep -q 'no tests to run' "$out/$name.log"; then |
| 242 | echo "--- $name matched no tests; its -run pattern has drifted from the list" |
| 243 | rc=1 |
| 244 | fi |
| 245 | ;; |
| 246 | esac |
| 247 | [ "$rc" -eq 0 ] || failed="$failed $name" |
| 248 | done |
| 249 | |
| 250 | if [ -n "$failed" ]; then |
| 251 | # The logs above can run to megabytes, and CI keeps only the tail, so |
| 252 | # a failing test's name used to scroll out of sight (main bcae2a72, |
| 253 | # 2026-10-25). Each failed job's verdict lines are repeated here, last: |
| 254 | # the test names, their file:line messages, a panic, and the tests a |
| 255 | # timeout reports as still running. |
| 256 | echo |
| 257 | echo "Failure summary:" |
| 258 | for name in $failed; do |
| 259 | echo "--- $name" |
| 260 | grep -E '^[[:space:]]*--- FAIL|^panic:|^FAIL[[:space:]]|^[[:space:]]+[A-Za-z0-9_./-]+_test\.go:[0-9]+:|^[[:space:]]*running tests:|^[[:space:]]+Test[A-Za-z0-9_/-]* \([0-9.]+[a-z]+\)$' "$out/$name.log" 2>/dev/null | head -60 || true |
| 261 | done |
| 262 | echo |
| 263 | echo "FAILED:$failed" |
| 264 | exit 1 |
| 265 | fi |
| 266 | |
amadan