| 1 | package main |
| 2 | |
| 3 | // carlos system geoip publish|status|off (geoip spec §4.2, §5.5). The |
| 4 | // strings here await Paul's copy review (plan 1, Task 13) before merge. |
| 5 | |
| 6 | import ( |
| 7 | "context" |
| 8 | "encoding/json" |
| 9 | "errors" |
| 10 | "flag" |
| 11 | "fmt" |
| 12 | "io" |
| 13 | "log" |
| 14 | "net/http" |
| 15 | "os" |
| 16 | "strings" |
| 17 | "time" |
| 18 | ) |
| 19 | |
| 20 | func runSystemGeoipVerb(rest []string) { |
| 21 | if len(rest) == 0 { |
| 22 | fmt.Fprint(os.Stderr, "system geoip: no verb given\n\n") |
| 23 | systemUsage() |
| 24 | os.Exit(2) |
| 25 | } |
| 26 | switch rest[0] { |
| 27 | case "publish": |
| 28 | runSystemGeoipPublish(rest[1:]) |
| 29 | case "status": |
| 30 | runSystemGeoipStatus(rest[1:]) |
| 31 | case "off": |
| 32 | runSystemGeoipOff(rest[1:]) |
| 33 | default: |
| 34 | fmt.Fprintf(os.Stderr, "system geoip: unknown verb %q\n\n", rest[0]) |
| 35 | systemUsage() |
| 36 | os.Exit(2) |
| 37 | } |
| 38 | } |
| 39 | |
| 40 | func runSystemGeoipPublish(args []string) { |
| 41 | fs := flag.NewFlagSet("system geoip publish", flag.ExitOnError) |
| 42 | consoleFlag := consoleOption(fs, "console origin (default: the only logged-in one)") |
| 43 | force := fs.Bool("force", false, "publish even if the database has under half the current one's nodes") |
| 44 | fs.Usage = flagUsage(fs, "<file.mmdb>") |
| 45 | fs.Parse(args) |
| 46 | if fs.NArg() != 1 { |
| 47 | fs.Usage() |
| 48 | os.Exit(2) |
| 49 | } |
| 50 | sess := requireDeployAPISession(*consoleFlag) |
| 51 | printUsing(stdout, sess) |
| 52 | ptr, err := sess.publishGeoip(context.Background(), fs.Arg(0), *force, stdout) |
| 53 | if err != nil { |
| 54 | log.Fatalf("carlos system geoip publish: %v", err) |
| 55 | } |
| 56 | fmt.Fprintf(stdout, "%s %s: %s, built %s. every edge picks it up within 12 minutes\n", |
| 57 | colorFor(stdout).ok("published"), short8cli(ptr.SHA256), ptr.DatabaseType, time.Unix(ptr.BuildEpoch, 0).UTC().Format("2006-01-02")) |
| 58 | } |
| 59 | |
| 60 | // geoipPointerView is the pointer as the CLI reads it back from a job. |
| 61 | type geoipPointerView struct { |
| 62 | SHA256 string `json:"sha256"` |
| 63 | DatabaseType string `json:"database_type"` |
| 64 | BuildEpoch int64 `json:"build_epoch"` |
| 65 | } |
| 66 | |
| 67 | // publishGeoip streams the file in one raw PUT (the console stages it and |
| 68 | // answers 202 with a job), then follows the job to its result. |
| 69 | func (s *apiSession) publishGeoip(ctx context.Context, path string, force bool, w io.Writer) (geoipPointerView, error) { |
| 70 | f, err := os.Open(path) |
| 71 | if err != nil { |
| 72 | return geoipPointerView{}, err |
| 73 | } |
| 74 | defer f.Close() |
| 75 | fi, err := f.Stat() |
| 76 | if err != nil { |
| 77 | return geoipPointerView{}, err |
| 78 | } |
| 79 | q := "" |
| 80 | if force { |
| 81 | q = "?force=1" |
| 82 | } |
| 83 | req, err := http.NewRequestWithContext(ctx, http.MethodPut, s.c.base+"/api/cli/system/geoip"+q, f) |
| 84 | if err != nil { |
| 85 | return geoipPointerView{}, err |
| 86 | } |
| 87 | req.ContentLength = fi.Size() |
| 88 | req.Header.Set("Content-Type", "application/octet-stream") |
| 89 | if s.c.token != "" { |
| 90 | req.Header.Set("Authorization", "Bearer "+s.c.token) |
| 91 | } |
| 92 | // The upload's own client, like doMultipartMethod's: c.httpc's 30s |
| 93 | // absolute timeout is sized for JSON, not a 130 MB database. |
| 94 | client := &http.Client{Timeout: shipHTTPClientTimeout, CheckRedirect: neverFollow} |
| 95 | res, err := client.Do(req) |
| 96 | if err != nil { |
| 97 | return geoipPointerView{}, err |
| 98 | } |
| 99 | defer res.Body.Close() |
| 100 | if merr := consoleMoved(s.c.base, res.StatusCode, res.Header.Get("Location")); merr != nil { |
| 101 | return geoipPointerView{}, offerConsoleMoved(merr) |
| 102 | } |
| 103 | body, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20)) |
| 104 | if res.StatusCode != http.StatusAccepted { |
| 105 | return geoipPointerView{}, apiErrorFromBody(res.StatusCode, body) |
| 106 | } |
| 107 | var started struct { |
| 108 | Job string `json:"job"` |
| 109 | } |
| 110 | if err := json.Unmarshal(body, &started); err != nil || started.Job == "" { |
| 111 | return geoipPointerView{}, errors.New("the console accepted the upload but named no job") |
| 112 | } |
| 113 | return s.watchGeoipJob(ctx, started.Job, w) |
| 114 | } |
| 115 | |
| 116 | func (s *apiSession) watchGeoipJob(ctx context.Context, id string, w io.Writer) (geoipPointerView, error) { |
| 117 | printed := 0 |
| 118 | deadline := time.Now().Add(16 * time.Minute) |
| 119 | for { |
| 120 | st, body, err := s.getJSON(ctx, "/api/cli/system/jobs/"+id) |
| 121 | if err != nil { |
| 122 | return geoipPointerView{}, fmt.Errorf("job %s: %v", id, err) |
| 123 | } |
| 124 | if st != http.StatusOK { |
| 125 | return geoipPointerView{}, fmt.Errorf("job %s: %s", id, apiErrorFromBody(st, body)) |
| 126 | } |
| 127 | var j struct { |
| 128 | Steps []string `json:"steps"` |
| 129 | Done bool `json:"done"` |
| 130 | Error string `json:"error"` |
| 131 | Result *geoipPointerView `json:"result"` |
| 132 | } |
| 133 | _ = json.Unmarshal(body, &j) |
| 134 | for ; printed < len(j.Steps); printed++ { |
| 135 | fmt.Fprintf(w, " %s\n", j.Steps[printed]) |
| 136 | } |
| 137 | if j.Done { |
| 138 | if j.Error != "" { |
| 139 | return geoipPointerView{}, errors.New(j.Error) |
| 140 | } |
| 141 | if j.Result == nil { |
| 142 | return geoipPointerView{}, errors.New("the job finished without naming what it published") |
| 143 | } |
| 144 | return *j.Result, nil |
| 145 | } |
| 146 | if time.Now().After(deadline) { |
| 147 | return geoipPointerView{}, errors.New("the console's job did not finish in 16 minutes") |
| 148 | } |
| 149 | time.Sleep(geoipPollInterval) |
| 150 | } |
| 151 | } |
| 152 | |
| 153 | // geoipPollInterval is a var so a test need not wait seconds per poll. |
| 154 | var geoipPollInterval = 2 * time.Second |
| 155 | |
| 156 | type geoipStatusResponse struct { |
| 157 | Pointer *struct { |
| 158 | SHA256 string `json:"sha256"` |
| 159 | DatabaseType string `json:"database_type"` |
| 160 | Build time.Time `json:"build"` |
| 161 | Published time.Time `json:"published"` |
| 162 | Origin string `json:"origin"` |
| 163 | Expired bool `json:"expired"` |
| 164 | ExpiresAt time.Time `json:"expires_at"` |
| 165 | Attribution string `json:"attribution"` |
| 166 | } `json:"pointer"` |
| 167 | PointerError string `json:"pointer_error"` |
| 168 | Feed *struct { |
| 169 | Name string `json:"name"` |
| 170 | LastSuccess time.Time `json:"last_success"` |
| 171 | Failures int `json:"failures"` |
| 172 | NotFound int `json:"not_found"` |
| 173 | LastFailure string `json:"last_failure"` |
| 174 | Failing bool `json:"failing"` |
| 175 | } `json:"feed"` |
| 176 | Boxes []struct { |
| 177 | Label string `json:"label"` |
| 178 | Fleet string `json:"fleet"` |
| 179 | State string `json:"state"` |
| 180 | } `json:"boxes"` |
| 181 | Ready bool `json:"ready"` |
| 182 | Blocking []string `json:"blocking"` |
| 183 | FeedSet bool `json:"feed_set"` |
| 184 | } |
| 185 | |
| 186 | func runSystemGeoipStatus(args []string) { |
| 187 | fs := flag.NewFlagSet("system geoip status", flag.ExitOnError) |
| 188 | consoleFlag := consoleOption(fs, "console origin (default: the only logged-in one)") |
| 189 | fs.Usage = flagUsage(fs, "") |
| 190 | fs.Parse(args) |
| 191 | sess := requireDeployAPISession(*consoleFlag) |
| 192 | st, body, err := sess.getJSON(context.Background(), "/api/cli/system/geoip") |
| 193 | if err != nil { |
| 194 | log.Fatalf("carlos system geoip status: %v", err) |
| 195 | } |
| 196 | if st != http.StatusOK { |
| 197 | log.Fatalf("carlos system geoip status: %s", apiErrorFromBody(st, body)) |
| 198 | } |
| 199 | var resp geoipStatusResponse |
| 200 | if err := json.Unmarshal(body, &resp); err != nil { |
| 201 | log.Fatalf("carlos system geoip status: unreadable response: %v", err) |
| 202 | } |
| 203 | printGeoipStatus(stdout, resp, time.Now()) |
| 204 | } |
| 205 | |
| 206 | // printGeoipStatus is the verb minus the fetch, so it is testable. |
| 207 | func printGeoipStatus(w io.Writer, resp geoipStatusResponse, now time.Time) { |
| 208 | c := colorFor(w) |
| 209 | switch p := resp.Pointer; { |
| 210 | case resp.PointerError != "": |
| 211 | fmt.Fprintf(w, "database: %s\n", c.bad(resp.PointerError)) |
| 212 | case p == nil: |
| 213 | fmt.Fprintln(w, "database: none published. geoip is off") |
| 214 | case p.Expired: |
| 215 | fmt.Fprintf(w, "database: %s %s expired. publish a newer one or turn geoip off\n", short8cli(p.SHA256), p.DatabaseType) |
| 216 | default: |
| 217 | age := int(now.Sub(p.Build).Hours() / 24) |
| 218 | fmt.Fprintf(w, "database: %s %s, built %s (%d days ago), %s\n", short8cli(p.SHA256), p.DatabaseType, p.Build.Format("2006-01-02"), age, p.Origin) |
| 219 | if !p.ExpiresAt.IsZero() { |
| 220 | fmt.Fprintf(w, " expires %s (GeoLite2 must be replaced within 30 days of its build)\n", p.ExpiresAt.Format("2006-01-02")) |
| 221 | } |
| 222 | if p.Attribution != "" { |
| 223 | fmt.Fprintf(w, " pages that show this data must credit: %s\n", p.Attribution) |
| 224 | } |
| 225 | } |
| 226 | switch f := resp.Feed; { |
| 227 | case f != nil && (f.Failures > 0 || f.NotFound > 0): |
| 228 | n := f.Failures |
| 229 | if f.NotFound > n { |
| 230 | n = f.NotFound |
| 231 | } |
| 232 | line := fmt.Sprintf("feed: %s, failing (%d in a row, last: %s)", f.Name, n, f.LastFailure) |
| 233 | if f.Failing { |
| 234 | line = c.bad(line) |
| 235 | } |
| 236 | fmt.Fprintln(w, line) |
| 237 | case f != nil && !f.LastSuccess.IsZero(): |
| 238 | fmt.Fprintf(w, "feed: %s, last success %s\n", f.Name, f.LastSuccess.Format("2006-01-02")) |
| 239 | case resp.FeedSet: |
| 240 | fmt.Fprintln(w, "feed: set (CONSOLE_GEOIP_FEED), not run yet") |
| 241 | } |
| 242 | fmt.Fprintln(w) |
| 243 | for _, b := range resp.Boxes { |
| 244 | name := b.Label |
| 245 | if b.Fleet != "" { |
| 246 | name = b.Fleet + "/" + b.Label |
| 247 | } |
| 248 | fmt.Fprintf(w, " %-28s %s\n", name, b.State) |
| 249 | } |
| 250 | fmt.Fprintln(w) |
| 251 | if resp.Ready { |
| 252 | fmt.Fprintf(w, "%s: every box strips forged headers and has the current database\n", c.ok("ready")) |
| 253 | return |
| 254 | } |
| 255 | fmt.Fprintf(w, "%s: %d in the way\n", c.bad("not ready"), len(resp.Blocking)) |
| 256 | for _, b := range resp.Blocking { |
| 257 | fmt.Fprintf(w, " %s\n", b) |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | func runSystemGeoipOff(args []string) { |
| 262 | fs := flag.NewFlagSet("system geoip off", flag.ExitOnError) |
| 263 | consoleFlag := consoleOption(fs, "console origin (default: the only logged-in one)") |
| 264 | fs.Usage = flagUsage(fs, "") |
| 265 | fs.Parse(args) |
| 266 | sess := requireDeployAPISession(*consoleFlag) |
| 267 | st, body, err := sess.deleteJSON(context.Background(), "/api/cli/system/geoip") |
| 268 | if err != nil { |
| 269 | log.Fatalf("carlos system geoip off: %v", err) |
| 270 | } |
| 271 | if st != http.StatusOK { |
| 272 | log.Fatalf("carlos system geoip off: %s", apiErrorFromBody(st, body)) |
| 273 | } |
| 274 | var out struct { |
| 275 | FeedSet bool `json:"feed_set"` |
| 276 | } |
| 277 | _ = json.Unmarshal(body, &out) |
| 278 | printGeoipOff(stdout, out.FeedSet) |
| 279 | } |
| 280 | |
| 281 | func printGeoipOff(w io.Writer, feedSet bool) { |
| 282 | fmt.Fprintln(w, "geoip is off. every edge drops the database within 12 minutes") |
| 283 | if feedSet { |
| 284 | fmt.Fprintln(w, "the feed is still set and will publish again within a day. unset CONSOLE_GEOIP_FEED to keep it off") |
| 285 | } |
| 286 | } |
| 287 | |
| 288 | func short8cli(sha string) string { |
| 289 | if len(sha) > 8 { |
| 290 | return sha[:8] |
| 291 | } |
| 292 | return strings.TrimSpace(sha) |
| 293 | } |
| 294 | |
amadan