| 1 | // canary_region_e2e_test.go |
| 2 | package main |
| 3 | |
| 4 | import ( |
| 5 | "context" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "os" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/console" |
| 14 | "github.com/carlosframework/platform/internal/provision" |
| 15 | "github.com/carlosframework/platform/internal/release" |
| 16 | ) |
| 17 | |
| 18 | // hasRefusalLine reports whether a whole output line is want, behind at most |
| 19 | // the log package's own timestamp. A swap of two adjacent arguments in the |
| 20 | // sentence is a different line and fails. |
| 21 | func hasRefusalLine(out, want string) bool { |
| 22 | for _, l := range strings.Split(out, "\n") { |
| 23 | l = strings.TrimSpace(l) |
| 24 | if l == want { |
| 25 | return true |
| 26 | } |
| 27 | // "2006/01/02 15:04:05 " is 20 bytes. |
| 28 | if len(l) > 20 && l[4] == '/' && l[7] == '/' && l[10] == ' ' && l[13] == ':' && l[16] == ':' && l[19] == ' ' && l[20:] == want { |
| 29 | return true |
| 30 | } |
| 31 | } |
| 32 | return false |
| 33 | } |
| 34 | |
| 35 | // Two identical runs per case: the same refusal both times, and no write, |
| 36 | // no new record, no manifest and no upload, before or after. |
| 37 | func TestCanaryRegionRefusesBeforeAnyWrite(t *testing.T) { |
| 38 | const existing = "probe-titogo.bab.oncarlos.test" |
| 39 | cases := []struct { |
| 40 | name, mode, region string |
| 41 | fleet, canary bool |
| 42 | ahead int // records whose hosts sort before the canary's |
| 43 | unreadable bool // the canary's record cannot be read |
| 44 | want string |
| 45 | }{ |
| 46 | {"fleet-placed app", "", "uk", true, false, 0, false, canaryRegionOnFleet("titogo", "acme")}, |
| 47 | {"older console", "legacy", "uk", true, false, 0, false, canaryFleetUnknown("titogo")}, |
| 48 | {"placement unknown", "placement-unknown", "uk", false, false, 0, false, canaryFleetUnknown("titogo")}, |
| 49 | {"existing canary elsewhere", "", "ie", false, true, 0, false, canaryRegionMismatch(existing, "uk")}, |
| 50 | // codex plan-R3 1: past the list's 500-record cap, and unreadable |
| 51 | // (a list skips it), the canary's own record still decides. |
| 52 | {"existing canary past the list cap", "", "ie", false, true, 520, false, canaryRegionMismatch(existing, "uk")}, |
| 53 | {"existing canary unreadable", "", "ie", false, true, 0, true, canaryRegionUnconfirmed("probe")}, |
| 54 | // codex plan-R4 1: a fleet-placed app whose stored region the console |
| 55 | // could not read; R9a leaves effective empty, and the flag is refused. |
| 56 | {"fleet, region unknown", "region-unknown", "uk", true, false, 0, false, canaryFleetUnknown("titogo")}, |
| 57 | } |
| 58 | for _, c := range cases { |
| 59 | t.Run(c.name, func(t *testing.T) { |
| 60 | e := regionE2EConsole(t, "") |
| 61 | ctx := context.Background() |
| 62 | if c.fleet { |
| 63 | if err := e.cst.SetAppPlacement(console.OpsAccountID, "titogo", "fleet/acme"); err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | } |
| 67 | for i := 0; i < c.ahead; i++ { |
| 68 | h := fmt.Sprintf("a%03d.titogo.bab.oncarlos.test", i) |
| 69 | rec := provision.Record{Schema: 1, Host: h, App: "titogo", Account: release.OpsAccount, |
| 70 | Channel: "stable", CreatedAt: time.Now().UTC()} |
| 71 | if err := provision.CreateRecord(ctx, e.store, rec); err != nil { |
| 72 | t.Fatal(err) |
| 73 | } |
| 74 | } |
| 75 | if c.canary { |
| 76 | rec := provision.Record{Schema: 1, Host: existing, App: "titogo", Account: release.OpsAccount, |
| 77 | Region: "uk", Channel: "canary/probe", CreatedAt: time.Now().UTC()} |
| 78 | if err := provision.CreateRecord(ctx, e.store, rec); err != nil { |
| 79 | t.Fatal(err) |
| 80 | } |
| 81 | } |
| 82 | if c.unreadable { |
| 83 | e.failGet.Store(provision.RecordKey(release.OpsAccount, "titogo", existing)) |
| 84 | } |
| 85 | e.mode.Store(c.mode) |
| 86 | for run := 0; run < 2; run++ { |
| 87 | out, err := e.run(t, "canary", "deploy", "--app", "titogo", "--name", "probe", "--region", c.region, |
| 88 | "--console", e.url, "--version", "abc1234", "--timeout", "1s", artifact(t)) |
| 89 | if err == nil || !hasRefusalLine(out, "canary: "+c.want) { |
| 90 | t.Fatalf("run %d: err=%v out=%s, want the whole line %q", run, err, out, "canary: "+c.want) |
| 91 | } |
| 92 | } |
| 93 | recs, _, err := provision.ListRecords(ctx, e.store, release.OpsAccount, "titogo", 1000) |
| 94 | if err != nil { |
| 95 | t.Fatal(err) |
| 96 | } |
| 97 | wantRecs := c.ahead |
| 98 | if c.canary { |
| 99 | wantRecs++ |
| 100 | } |
| 101 | if len(recs) != wantRecs || e.writes.Load() != 0 || e.releases(t) != 0 { |
| 102 | t.Fatalf("records %d (want %d), writes %d, releases %d; want nothing written", len(recs), wantRecs, e.writes.Load(), e.releases(t)) |
| 103 | } |
| 104 | if c.canary { |
| 105 | rec, err := provision.GetRecord(ctx, e.store, release.OpsAccount, "titogo", existing) |
| 106 | if err != nil || rec.Region != "uk" { |
| 107 | t.Fatalf("the existing canary = %+v %v, want it still in uk", rec, err) |
| 108 | } |
| 109 | } |
| 110 | if _, err := provision.GetManifest(ctx, e.store, release.OpsAccount, "titogo"); !errors.Is(err, os.ErrNotExist) { |
| 111 | t.Fatalf("an instances manifest was written: %v", err) |
| 112 | } |
| 113 | }) |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | // codex plan-R5 1: a same-region canary past the list's 500-record cap is |
| 118 | // the record the command reuses, found by the direct read, never re-created. |
| 119 | // On its own channel it is left as it is; on another it is rechannelled. |
| 120 | // Either way there is no create request and the run goes on to ship. |
| 121 | func TestCanaryRegionReusesACanaryPastTheListCap(t *testing.T) { |
| 122 | const existing = "probe-titogo.bab.oncarlos.test" |
| 123 | for _, c := range []struct{ name, channel, env string }{ |
| 124 | {"same channel", "canary/probe", ""}, |
| 125 | {"different channel", "canary/old", ""}, |
| 126 | // The environment bind follows the same decision: an existing |
| 127 | // record gets its own bind write, never a create carrying it. |
| 128 | {"same channel, with --environment", "canary/probe", "beta"}, |
| 129 | } { |
| 130 | t.Run(c.name, func(t *testing.T) { |
| 131 | e := regionE2EConsole(t, "") |
| 132 | ctx := context.Background() |
| 133 | for i := 0; i < 520; i++ { |
| 134 | rec := provision.Record{Schema: 1, Host: fmt.Sprintf("a%03d.titogo.bab.oncarlos.test", i), App: "titogo", |
| 135 | Account: release.OpsAccount, Channel: "stable", CreatedAt: time.Now().UTC()} |
| 136 | if err := provision.CreateRecord(ctx, e.store, rec); err != nil { |
| 137 | t.Fatal(err) |
| 138 | } |
| 139 | } |
| 140 | rec := provision.Record{Schema: 1, Host: existing, App: "titogo", Account: release.OpsAccount, |
| 141 | Region: "uk", Channel: c.channel, CreatedAt: time.Now().UTC()} |
| 142 | if err := provision.CreateRecord(ctx, e.store, rec); err != nil { |
| 143 | t.Fatal(err) |
| 144 | } |
| 145 | args := []string{"canary", "deploy", "--app", "titogo", "--name", "probe", "--region", "uk", |
| 146 | "--console", e.url, "--version", "abc1234", "--timeout", "1s"} |
| 147 | if c.env != "" { |
| 148 | args = append(args, "--environment", c.env) |
| 149 | } |
| 150 | out, _ := e.run(t, append(args, artifact(t))...) |
| 151 | if n := e.creates.Load(); n != 0 { |
| 152 | t.Fatalf("%d create request(s) for a canary that exists:\n%s", n, out) |
| 153 | } |
| 154 | if e.releases(t) == 0 { |
| 155 | t.Fatalf("the run never shipped:\n%s", out) |
| 156 | } |
| 157 | got, err := provision.GetRecord(ctx, e.store, release.OpsAccount, "titogo", existing) |
| 158 | if err != nil || got.Region != "uk" || got.Channel != "canary/probe" || got.Environment != c.env { |
| 159 | t.Fatalf("the canary = %+v %v, want it in uk on canary/probe, environment %q", got, err, c.env) |
| 160 | } |
| 161 | if recs, _, err := provision.ListRecords(ctx, e.store, release.OpsAccount, "titogo", 1000); err != nil || len(recs) != 521 { |
| 162 | t.Fatalf("records = %d %v, want the 521 there were", len(recs), err) |
| 163 | } |
| 164 | }) |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | // codex plan-R3 6: a successful replica create prints C14 with its place, |
| 169 | // not an older console's "(channel X)". This console has no secrets key, |
| 170 | // so no box is eligible and the place falls back to its code. |
| 171 | func TestInstancesCreateReplicaPrintsItsPlace(t *testing.T) { |
| 172 | e := regionE2EConsole(t, "") |
| 173 | ctx := context.Background() |
| 174 | m := provision.Manifest{Schema: 1, App: "titogo", Domain: "bab.oncarlos.test", Channel: "beta"} |
| 175 | if err := provision.PutManifest(ctx, e.store, release.OpsAccount, "titogo", m); err != nil { |
| 176 | t.Fatal(err) |
| 177 | } |
| 178 | box := `{"v":1,"box":"uk-box","eip":"1.2.3.4","region":"eu-west-2","placement":"uk","hibernation_configured":true,"hosts":{}}` |
| 179 | if _, err := e.store.Put(ctx, "control/edges/uk-box.json", []byte(box), ""); err != nil { |
| 180 | t.Fatal(err) |
| 181 | } |
| 182 | const host = "r.bab.oncarlos.test" |
| 183 | out, err := e.run(t, "instances", "create", "--app", "titogo", "--host", host, "--region", "uk", "--role", "replica", "--console", e.url) |
| 184 | if err != nil { |
| 185 | t.Fatalf("create: %v\n%s", err, out) |
| 186 | } |
| 187 | if want := "created instance " + host + " (" + instanceCreatedSuffix("beta", "uk") + ")"; !hasRefusalLine(out, want) { |
| 188 | t.Fatalf("stdout = %q, want the whole line %q", out, want) |
| 189 | } |
| 190 | } |
| 191 | |
amadan