amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package console
2
3import (
4 "archive/tar"
5 "bytes"
6 "compress/gzip"
7 "context"
8 "crypto/sha256"
9 "crypto/tls"
10 "encoding/hex"
11 "encoding/json"
12 "errors"
13 "io"
14 "net/http"
15 "net/http/httptest"
16 "os"
17 "strings"
18 "sync/atomic"
19 "testing"
20 "time"
21
22 "github.com/carlosframework/platform/internal/geoip"
23 "github.com/carlosframework/platform/internal/release"
24)
25
26func gzipBytes(b []byte) []byte {
27 var buf bytes.Buffer
28 w := gzip.NewWriter(&buf)
29 w.Write(b)
30 w.Close()
31 return buf.Bytes()
32}
33
34func tarGzBytes(name string, body []byte) []byte {
35 var buf bytes.Buffer
36 zw := gzip.NewWriter(&buf)
37 tw := tar.NewWriter(zw)
38 tw.WriteHeader(&tar.Header{Name: "GeoLite2-City_20261006/COPYRIGHT.txt", Mode: 0o644, Size: 5, Typeflag: tar.TypeReg})
39 tw.Write([]byte("(c) x"))
40 tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})
41 tw.Write(body)
42 tw.Close()
43 zw.Close()
44 return buf.Bytes()
45}
46
47var feedNow = time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
48
49func newTestFeeder(t *testing.T, provider string, h http.Handler) (*geoipFeeder, release.Store) {
50 t.Helper()
51 srv := httptest.NewTLSServer(h)
52 t.Cleanup(srv.Close)
53 p, s := newTestPublisher(t)
54 f := &geoipFeeder{
55 pub: p, store: s, spec: geoip.FeedSpec{Name: "test-feed", Type: "GeoIP2-City", Provider: provider},
56 account: "acct", key: "s3cret-key", box: "console-box",
57 client: &http.Client{Timeout: time.Minute, CheckRedirect: geoipFeedRedirect,
58 Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}},
59 now: func() time.Time { return feedNow }, maxExpanded: geoipFeedMaxExpanded,
60 dbipBase: srv.URL, maxmindBase: srv.URL,
61 }
62 return f, s
63}
64
65func feedStatus(t *testing.T, s release.Store) geoip.FeedStatus {
66 t.Helper()
67 b, _, err := s.Get(context.Background(), geoip.FeedKey)
68 if err != nil {
69 t.Fatalf("feed.json: %v", err)
70 }
71 st, err := geoip.ParseFeedStatus(b)
72 if err != nil {
73 t.Fatal(err)
74 }
75 return st
76}
77
78func TestFeedDBIPPublishesThisMonthThenRests(t *testing.T) {
79 withFixtureProbes(t)
80 body := gzipBytes(fixture(t, "GeoIP2-City-Test.mmdb"))
81 var gets atomic.Int32
82 f, s := newTestFeeder(t, "dbip", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
83 gets.Add(1)
84 if r.URL.Path != "/free/dbip-test-feed-lite-2026-10.mmdb.gz" && !strings.HasSuffix(r.URL.Path, "-2026-10.mmdb.gz") {
85 http.NotFound(w, r)
86 return
87 }
88 w.Write(body)
89 }))
90 if err := f.Check(context.Background()); err != nil {
91 t.Fatal(err)
92 }
93 cur, _, present, _ := f.pub.Current(context.Background())
94 if !present || cur.Origin != "feed:test-feed" || cur.Edition != "2026-10" {
95 t.Fatalf("pointer = %+v", cur)
96 }
97 st := feedStatus(t, s)
98 if st.LastEdition != "2026-10" || st.ConsecutiveFailures != 0 || st.Box != "console-box" {
99 t.Fatalf("feed.json = %+v", st)
100 }
101 before := gets.Load()
102 if err := f.Check(context.Background()); err != nil {
103 t.Fatal(err)
104 }
105 if gets.Load() != before {
106 t.Fatal("a second check in the same month downloaded again")
107 }
108}
109
110func TestFeedDBIPFallsBackToLastMonthOnlyWithoutAPointerOfItsOwn(t *testing.T) {
111 withFixtureProbes(t)
112 body := gzipBytes(fixture(t, "GeoIP2-City-Test.mmdb"))
113 f, _ := newTestFeeder(t, "dbip", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
114 if strings.HasSuffix(r.URL.Path, "-2026-09.mmdb.gz") {
115 w.Write(body)
116 return
117 }
118 http.NotFound(w, r)
119 }))
120 if err := f.Check(context.Background()); err != nil {
121 t.Fatal(err)
122 }
123 if cur, _, _, _ := f.pub.Current(context.Background()); cur.Edition != "2026-09" {
124 t.Fatalf("edition = %q, want last month's", cur.Edition)
125 }
126}
127
128func TestFeedNotFoundCountsTowardTheAlert(t *testing.T) {
129 f, s := newTestFeeder(t, "dbip", http.NotFoundHandler())
130 for i := 1; i <= 7; i++ {
131 if err := f.Check(context.Background()); !errors.Is(err, errFeedNotFound) {
132 t.Fatalf("check %d = %v", i, err)
133 }
134 st := feedStatus(t, s)
135 if st.ConsecutiveNotFound != i || st.Failing() != (i >= 7) {
136 t.Fatalf("after %d checks feed.json = %+v", i, st)
137 }
138 }
139}
140
141func maxmindHandler(t *testing.T, archive []byte, sum string, lastModified string, gets *atomic.Int32) http.Handler {
142 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
143 user, pass, ok := r.BasicAuth()
144 if !ok || user != "acct" || pass != "s3cret-key" {
145 http.Error(w, "no auth", http.StatusUnauthorized)
146 return
147 }
148 if strings.Contains(r.URL.String(), "s3cret") {
149 t.Error("the key appeared in a URL")
150 }
151 w.Header().Set("Last-Modified", lastModified)
152 if r.Method == http.MethodHead {
153 return
154 }
155 gets.Add(1)
156 switch r.URL.Query().Get("suffix") {
157 case "tar.gz.sha256":
158 w.Write([]byte(sum + " GeoLite2-City_20261006.tar.gz\n"))
159 case "tar.gz":
160 w.Write(archive)
161 default:
162 http.NotFound(w, r)
163 }
164 })
165}
166
167func TestFeedMaxMindHeadThenChecksumThenArchive(t *testing.T) {
168 withFixtureProbes(t)
169 archive := tarGzBytes("GeoLite2-City_20261006/GeoLite2-City.mmdb", fixture(t, "GeoIP2-City-Test.mmdb"))
170 sum := sha256.Sum256(archive)
171 var gets atomic.Int32
172 f, _ := newTestFeeder(t, "maxmind", maxmindHandler(t, archive, hex.EncodeToString(sum[:]), "Tue, 06 Oct 2026 00:00:00 GMT", &gets))
173 if err := f.Check(context.Background()); err != nil {
174 t.Fatal(err)
175 }
176 cur, _, _, _ := f.pub.Current(context.Background())
177 if cur.Edition != "Tue, 06 Oct 2026 00:00:00 GMT" {
178 t.Fatalf("edition = %q", cur.Edition)
179 }
180 before := gets.Load()
181 if err := f.Check(context.Background()); err != nil {
182 t.Fatal(err)
183 }
184 if gets.Load() != before {
185 t.Fatal("an unchanged Last-Modified still downloaded")
186 }
187}
188
189func TestFeedMaxMindWrongChecksumPublishesNothing(t *testing.T) {
190 withFixtureProbes(t)
191 archive := tarGzBytes("x/GeoLite2-City.mmdb", fixture(t, "GeoIP2-City-Test.mmdb"))
192 var gets atomic.Int32
193 f, s := newTestFeeder(t, "maxmind", maxmindHandler(t, archive, strings.Repeat("0", 64), "Tue, 06 Oct 2026 00:00:00 GMT", &gets))
194 if err := f.Check(context.Background()); err == nil {
195 t.Fatal("a bad checksum published")
196 }
197 if _, _, present, _ := f.pub.Current(context.Background()); present {
198 t.Fatal("pointer written despite the checksum")
199 }
200 if st := feedStatus(t, s); st.LastFailure != "checksum" || st.ConsecutiveFailures != 1 {
201 t.Fatalf("feed.json = %+v", st)
202 }
203 if entries, _ := os.ReadDir(f.pub.tmpRoot); len(entries) != 0 {
204 t.Fatalf("staging left behind: %v", entries)
205 }
206}
207
208func TestFeedRedirectDropsCredentialsAcrossHosts(t *testing.T) {
209 var sawAuth atomic.Bool
210 other := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
211 if r.Header.Get("Authorization") != "" {
212 sawAuth.Store(true)
213 }
214 w.WriteHeader(http.StatusTeapot)
215 }))
216 t.Cleanup(other.Close)
217 f, _ := newTestFeeder(t, "maxmind", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
218 http.Redirect(w, r, other.URL+"/presigned", http.StatusFound)
219 }))
220 res, err := f.get(context.Background(), f.maxmindBase+"/x", true)
221 if err != nil {
222 t.Fatal(err)
223 }
224 res.Body.Close()
225 if sawAuth.Load() {
226 t.Fatal("the MaxMind credentials followed a redirect to another host")
227 }
228 loop := 0
229 f2, _ := newTestFeeder(t, "maxmind", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
230 loop++
231 http.Redirect(w, r, "/again", http.StatusFound)
232 }))
233 if _, err := f2.get(context.Background(), f2.maxmindBase+"/x", true); err == nil {
234 t.Fatal("an endless redirect was followed")
235 }
236}
237
238func TestFeedRefusesADecompressionBomb(t *testing.T) {
239 var buf bytes.Buffer
240 zw := gzip.NewWriter(&buf)
241 zero := make([]byte, 1<<20)
242 for i := 0; i < 8; i++ {
243 zw.Write(zero)
244 }
245 zw.Close()
246 bomb := buf.Bytes()
247 f, s := newTestFeeder(t, "dbip", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write(bomb) }))
248 f.maxExpanded = 1 << 20
249 if err := f.Check(context.Background()); err == nil {
250 t.Fatal("a bomb was accepted")
251 }
252 if st := feedStatus(t, s); st.LastFailure != "too large" {
253 t.Fatalf("feed.json = %+v", st)
254 }
255 if entries, _ := os.ReadDir(f.pub.tmpRoot); len(entries) != 0 {
256 t.Fatalf("staging left behind: %v", entries)
257 }
258}
259
260func TestFeedRefusesTheWrongProductEvenFirstTime(t *testing.T) {
261 withFixtureProbes(t)
262 body := gzipBytes(fixture(t, "GeoIP2-Country-Test.mmdb")) // feed says City
263 f, s := newTestFeeder(t, "dbip", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write(body) }))
264 if err := f.Check(context.Background()); err == nil {
265 t.Fatal("a Country database was published by a City feed")
266 }
267 if st := feedStatus(t, s); st.LastFailure != "wrong type" {
268 t.Fatalf("feed.json = %+v", st)
269 }
270}
271
272func TestFeedSkipsWhileSomeoneIsPublishing(t *testing.T) {
273 body := gzipBytes(fixture(t, "GeoIP2-City-Test.mmdb"))
274 f, s := newTestFeeder(t, "dbip", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write(body) }))
275 f.pub.mu.Lock()
276 err := f.Check(context.Background())
277 f.pub.mu.Unlock()
278 if !errors.Is(err, errGeoipBusy) {
279 t.Fatalf("check while busy = %v", err)
280 }
281 if st := feedStatus(t, s); st.ConsecutiveFailures != 0 {
282 t.Fatalf("a busy slot counted as a failure: %+v", st)
283 }
284}
285
286// Code review R2-2: a check that used up its deadline still records the
287// failure.
288func TestFeedRecordsAFailureAfterItsDeadline(t *testing.T) {
289 f, s := newTestFeeder(t, "dbip", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
290 select {
291 case <-r.Context().Done():
292 case <-time.After(5 * time.Second):
293 }
294 }))
295 ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
296 defer cancel()
297 if err := f.Check(ctx); err == nil {
298 t.Fatal("a stalled provider succeeded")
299 }
300 if st := feedStatus(t, s); st.ConsecutiveFailures != 1 || st.LastFailure != "download" {
301 t.Fatalf("feed.json after a timeout = %+v", st)
302 }
303}
304
305// Code review R2-1: an oversized feed.json is replaced, never read whole.
306func TestFeedReplacesAnOversizedStatus(t *testing.T) {
307 f, s := newTestFeeder(t, "dbip", http.NotFoundHandler())
308 s.Put(context.Background(), geoip.FeedKey, bytes.Repeat([]byte("x"), geoip.MaxPointerBytes+10), "")
309 f.Check(context.Background())
310 if st := feedStatus(t, s); st.ConsecutiveNotFound != 1 {
311 t.Fatalf("feed.json = %+v", st)
312 }
313}
314
315type flakyFeedStore struct {
316 release.Store
317 failOpen atomic.Bool
318}
319
320func (s *flakyFeedStore) Open(ctx context.Context, key string) (io.ReadCloser, string, error) {
321 if key == geoip.FeedKey && s.failOpen.Load() {
322 return nil, "", errors.New("transient")
323 }
324 return release.OpenObject(ctx, s.Store, key)
325}
326
327// Code review R3-2: an unreadable feed.json is never overwritten.
328func TestFeedNeverOverwritesAnUnreadableHistory(t *testing.T) {
329 f, s := newTestFeeder(t, "dbip", http.NotFoundHandler())
330 raw, _ := json.Marshal(geoip.FeedStatus{V: 1, Feed: f.spec.Name, Box: "b", ConsecutiveFailures: 7})
331 s.Put(context.Background(), geoip.FeedKey, raw, "")
332 flaky := &flakyFeedStore{Store: s}
333 flaky.failOpen.Store(true)
334 f.store = flaky
335 f.Check(context.Background())
336 if st := feedStatus(t, s); st.ConsecutiveFailures != 7 {
337 t.Fatalf("history rewritten after an unreadable read: %+v", st)
338 }
339}
340
341type failingDeleteStore struct {
342 release.Store
343 fails atomic.Int32
344}
345
346func (s *failingDeleteStore) Delete(ctx context.Context, key string) error {
347 if s.fails.Add(-1) >= 0 {
348 return errors.New("transient")
349 }
350 return s.Store.Delete(ctx, key)
351}
352
353// Code review R6-1: a failed clear is retried until it lands.
354func TestRetryClearFeedStatusRecoversWithoutARestart(t *testing.T) {
355 base := release.NewDirStore(t.TempDir())
356 base.Put(context.Background(), geoip.FeedKey, []byte(`{"v":1}`), "")
357 s := &failingDeleteStore{Store: base}
358 s.fails.Store(2)
359 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
360 defer cancel()
361 retryClearFeedStatus(ctx, s, time.Millisecond, 4*time.Millisecond)
362 if _, _, err := base.Get(context.Background(), geoip.FeedKey); !errors.Is(err, os.ErrNotExist) {
363 t.Fatalf("feed.json survived the retries: %v", err)
364 }
365}
366
367// Code review R7-1: a Last-Modified that is not an HTTP date never becomes
368// an edition, and the published generation survives.
369func TestFeedRefusesAnInvalidEdition(t *testing.T) {
370 withFixtureProbes(t)
371 archive := tarGzBytes("x/GeoLite2-City.mmdb", fixture(t, "GeoIP2-City-Test.mmdb"))
372 sum := sha256.Sum256(archive)
373 var gets atomic.Int32
374 f, s := newTestFeeder(t, "maxmind", maxmindHandler(t, archive, hex.EncodeToString(sum[:]), strings.Repeat("x", 70<<10), &gets))
375 good, err := f.pub.Publish(context.Background(), bytes.NewReader(fixture(t, "GeoIP2-Country-Test.mmdb")), geoipPublishOpts{Origin: "manual"})
376 if err != nil {
377 t.Fatal(err)
378 }
379 if err := f.Check(context.Background()); err == nil {
380 t.Fatal("a junk Last-Modified was accepted")
381 }
382 if cur, _, _, _ := f.pub.Current(context.Background()); cur.SHA256 != good.SHA256 {
383 t.Fatal("the usable generation was replaced")
384 }
385 if gets.Load() != 0 {
386 t.Fatal("downloaded on a junk Last-Modified")
387 }
388 _ = s
389}
390
391// Code review R7-2: re-enabling a feed whose edition is already published
392// records a success, not "not run yet".
393func TestFeedUnchangedEditionIsASuccess(t *testing.T) {
394 withFixtureProbes(t)
395 body := gzipBytes(fixture(t, "GeoIP2-City-Test.mmdb"))
396 f, s := newTestFeeder(t, "dbip", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write(body) }))
397 if err := f.Check(context.Background()); err != nil {
398 t.Fatal(err)
399 }
400 s.Delete(context.Background(), geoip.FeedKey) // disabled and cleared, then re-enabled
401 if err := f.Check(context.Background()); err != nil {
402 t.Fatal(err)
403 }
404 if st := feedStatus(t, s); st.LastSuccess.IsZero() || st.LastEdition != "2026-10" {
405 t.Fatalf("feed.json after an unchanged check = %+v", st)
406 }
407}
408