Deployment strategy — rolling, blue-green, canary আর feature flag

apiVersion: apps/v1
kind: Deployment
metadata:
  name: shop-blue
spec:
  replicas: 3
  selector:
    matchLabels: {app: shop, color: blue}
  template:
    metadata:
      labels: {app: shop, color: blue}
    spec:
      containers:
      - name: web
        image: nginx:1.27-alpine
        env:
        - {name: VERSION, value: v1}
        command: ["sh", "-c"]
        args:
        - echo "$VERSION" > /usr/share/nginx/html/index.html;
          exec nginx -g "daemon off;"
        readinessProbe:
          httpGet: {path: /, port: 80}
          periodSeconds: 2
        resources:
          requests: {cpu: 10m, memory: 16Mi}
          limits: {cpu: 100m, memory: 32Mi}
---
# shop-green is the same with color: green and VERSION v2
apiVersion: v1
kind: Service
metadata:
  name: shop
spec:
  selector: {app: shop, color: blue}
  ports:
  - {port: 80, targetPort: 80}

$ kubectl get svc shop -o wide
NAME   TYPE        CLUSTER-IP    …   PORT(S)   SELECTOR
shop   ClusterIP   10.101.71.3   …   80/TCP    app=shop,color=blue

# client is running; now, from another terminal:
$ kubectl patch service shop -p '{"spec":{"selector":{"app":"shop","color":"green"}}}'
service/shop patched

# client output: requests per second, by version
sec        v1        v2
  3        90         0
  4        20        69
  5         0        90
…
total 1068 {'v1': 376, 'v2': 692}
v1: first 0.000s last 4.223s
v2: first 4.234s last 11.994s

# back to blue, same command, client running again
total 1082 {'v1': 649, 'v2': 433}
v1: first 4.832s last 11.998s
v2: first 0.000s last 4.821s

$ kubectl get deployment shop-green
NAME         READY   UP-TO-DATE   AVAILABLE   AGE
shop-green   3/3     3            3           95s

sec   http500        v1
  1         0        91
  2        31        58
  3        90         0
…
  6        91         0
  7        61        29
  8         0        89
…
total 1261 {'http500': 454, 'v1': 807}
http500: first 2.654s last 7.675s

$ kubectl get deployment
NAME          READY   UP-TO-DATE   AVAILABLE   AGE
shop-canary   1/1     1            1           5s
shop-stable   9/9     9            9           5s

# new TCP connection per request, 15 seconds, three runs
RUN 1  2526 {'v1': 2282, 'v2': 244}
RUN 2  2552 {'v1': 2315, 'v2': 237}
RUN 3  2533 {'v1': 2295, 'v2': 238}

# one keep-alive connection for 2 seconds, six separate times
KEEP 1  366 {'v1': 366}
KEEP 2  371 {'v1': 371}
…
KEEP 6  365 {'v1': 365}

$ kubectl scale deployment shop-stable --replicas=4
deployment.apps/shop-stable scaled

RUN 1  2533 {'v1': 2042, 'v2': 491}
RUN 2  2548 {'v1': 2037, 'v2': 511}

apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: shop
spec:
  parentRefs:
  - name: shop-gw
  rules:
  - backendRefs:
    - {name: shop-stable, port: 80, weight: 99}
    - {name: shop-canary, port: 80, weight: 1}

$ kubectl set env deployment/web VERSION=v2
deployment.apps/web env updated

sec asset-404   pair-ok
  4         0        84
  5         4        82
  6        28        57
…
 10        44        40
 11        42        43
…
 19        30        54
 20         5        80
 21         0        85
…
total 3835 {'asset-404': 480, 'pair-ok': 3355}
asset-404: first 5.928s last 20.183s

// Canary sample size and error-rate confidence: a simple binomial model.
const zAlpha = 1.645, zPower = 0.842;      // one-sided 5%, power 80%

function sampleSize(p0, p1) {              // requests needed in EACH group
  const v = p0 * (1 - p0) + p1 * (1 - p1);
  return Math.ceil((zAlpha + zPower) ** 2 * v / (p1 - p0) ** 2);
}
function wilson(errors, n, z = 1.96) {     // 95% interval for an error rate
  const p = errors / n, d = 1 + z * z / n;
  const mid = (p + z * z / (2 * n)) / d;
  const h = z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d;
  return [Math.max(0, mid - h), Math.min(1, mid + h)];
}
const pct = x => (x * 100).toFixed(3) + "%";

console.log("baseline 0.5%: requests needed per group");
const rps = 500, share = 0.05;             // assumed traffic, canary share
for (const p1 of [0.02, 0.01, 0.0075, 0.006]) {
  const n = sampleSize(0.005, p1);
  const mins = n / (rps * share) / 60;
  console.log(`  detect ${pct(p1)}: n=${n}, ${mins.toFixed(1)} min at 5% of ${rps} rps`);
}
console.log("what a canary result can and cannot say (95% Wilson)");
for (const [e, n] of [[0, 100], [0, 1000], [1, 200], [3, 400], [12, 2000]]) {
  const [lo, hi] = wilson(e, n);
  console.log(`  ${e}/${n}: rate ${pct(e / n)}, interval ${pct(lo)} to ${pct(hi)}`);
}
console.log(`rule of three, 0/1000: upper bound about ${pct(3 / 1000)}`);

// Canary analysis with a seeded error stream: same seed, same result.
function mulberry32(a) {
  return () => {
    a |= 0; a = a + 0x6D2B79F5 | 0;
    let t = Math.imul(a ^ a >>> 15, 1 | a);
    t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
    return ((t ^ t >>> 14) >>> 0) / 4294967296;
  };
}
const gap = (rng, p) => Math.floor(Math.log(rng()) / Math.log(1 - p));
const binom = (rng, n, p) => { let k = 0;       // jump from error to error
  for (let i = gap(rng, p); i < n; i += 1 + gap(rng, p)) k++;
  return k; };

function analyse(canaryRate, seed) {
  const rng = mulberry32(seed), rps = 500;
  let fails = 0, log = "";
  for (const w of [0.01, 0.05, 0.25, 0.5]) {
    const n = Math.round(rps * w * 300), m = Math.round(rps * (1 - w) * 300);
    const ce = binom(rng, n, canaryRate), se = binom(rng, m, 0.005);
    const pool = (ce + se) / (n + m);
    const z = (ce / n - se / m) / Math.sqrt(pool * (1 - pool) * (1 / n + 1 / m));
    fails = z > 2.33 ? fails + 1 : 0;      // one-sided, about 1%
    log += `  ${w * 100}%: canary ${ce}/${n}, z=${z.toFixed(2)}\n`;
    if (fails >= 2) return log + "  ABORT, weight back to 0";
  }
  return log + "  PROMOTE to 100%";
}
for (const [name, rate] of [["healthy", 0.005], ["2x errors", 0.01], ["5x errors", 0.025]])
  console.log(`${name}, seed 42\n${analyse(rate, 42)}`);
const count = (rate, word) => { let c = 0;
  for (let s = 1; s <= 100; s++) if (analyse(rate, s).includes(word)) c++;
  return c; };
console.log(`healthy aborted: ${count(0.005, "ABORT")} of 100 seeds`);
console.log(`1.3x bug promoted: ${count(0.0065, "PROMOTE")} of 100 seeds`);

apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: shop
spec:
  replicas: 10
  selector:
    matchLabels: {app: shop}
  template:
    metadata:
      labels: {app: shop}
    spec:
      containers:
      - {name: web, image: "example/shop:2.0.0"}
  strategy:
    canary:
      steps:
      - setWeight: 5
      - pause: {duration: 5m}
      - analysis:
          templates:
          - templateName: error-rate
      - setWeight: 25
      - pause: {duration: 10m}
---
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: error-rate
spec:
  metrics:
  - name: error-rate
    interval: 1m
    count: 5
    failureLimit: 2
    successCondition: result[0] < 0.01
    provider:
      prometheus:
        address: http://prometheus.monitoring:9090
        query: |
          sum(rate(http_requests_total{app="shop",status=~"5.."}[1m]))
          / sum(rate(http_requests_total{app="shop"}[1m]))

// Expand-contract order checker. Each app version states what it needs.
const APP = {
  v1: { reads: "name",      writes: ["name"] },
  v2: { reads: "name",      writes: ["name", "full_name"] },
  v3: { reads: "full_name", writes: ["name", "full_name"] },
  v4: { reads: "full_name", writes: ["full_name"] },
};
function check(plan) {
  const cols = new Set(["name"]); let running = ["v1"], filled = false;
  for (const step of plan) {
    const [op, arg] = step.split(" ");
    if (op === "add") cols.add(arg);
    if (op === "drop") cols.delete(arg);
    if (op === "deploy") running = [running[running.length - 1], arg]; // mixed
    if (op === "finish") running = running.slice(-1);
    if (op === "backfill" && running.some(v => !APP[v].writes.includes("full_name")))
      return `FAIL "${step}": an old writer still leaves new rows empty`;
    if (op === "backfill") filled = true;
    for (const v of running) {
      for (const c of [APP[v].reads, ...APP[v].writes])
        if (!cols.has(c)) return `FAIL "${step}": ${v} needs missing column ${c}`;
      if (APP[v].reads === "full_name" && !filled)
        return `FAIL "${step}": ${v} reads full_name before backfill`;
    }
  }
  return "OK: every running version works at every step";
}
const good = ["add full_name", "deploy v2", "finish", "backfill", "deploy v3",
  "finish", "deploy v4", "finish", "drop name"];
const plans = { good, "rename at once": ["add full_name", "drop name", "deploy v4"],
  "backfill early": ["add full_name", "backfill", "deploy v2", "finish"],
  "read too early": ["add full_name", "deploy v2", "finish", "deploy v3"],
  "drop in rollout": good.slice(0, 7).concat("drop name") };
for (const [k, p] of Object.entries(plans)) console.log(k.padEnd(16), check(p));

sec  conn-err        v1        v2
  3         0        94         0
…
  5         0        94         0
  6         1        77        17
  7         0         0        94
…
total 1316 {'conn-err': 1, 'v1': 638, 'v2': 677}
v1: first 0.000s last 6.809s
v2: first 6.830s last 13.998s
errors {'RemoteDisconnected': 1}

// Cost and risk per strategy. All inputs are assumed round numbers.
const podHour = 0.05, rps = 1000;                  // $0.05 is an assumption
const releases = 20, badShare = 0.10, bugRate = 0.05;  // per month
const detectSec = 300;                             // alert needs 5 minutes

// downtime s, extra Pods, extra hours, share exposed, undo s
const S = [
  ["recreate",   60, 0,    0,    1.00, 60],
  ["rolling",     0, 2.5,  0.20, 0.50, 180],
  ["blue-green",  0, 10,   0.50, 1.00, 5],
  ["canary 5%",   0, 2,    1.00, 0.05, 30],
  ["canary+flag", 0, 2,    1.00, 0.05, 2],
];
console.log("strategy     $/month  planned-lost  bad-release-errors  month-total");
const rows = S.map(([name, down, extra, hours, share, undo]) => {
  const dollars = extra * hours * podHour * releases;
  const planned = down * rps * releases;                // every release
  const perBad = rps * share * bugRate * (detectSec + undo);
  const total = planned + perBad * releases * badShare;
  return { name, dollars, planned, perBad, total };
});
for (const r of rows) {
  console.log(r.name.padEnd(12), r.dollars.toFixed(2).padStart(7),
    String(r.planned).padStart(13), String(Math.round(r.perBad)).padStart(19),
    String(Math.round(r.total)).padStart(12));
}
const budget = rps * 86400 * 30 * 0.001;                // 99.9% SLO
console.log(`monthly error budget at 99.9%: ${budget} requests`);
for (const r of rows) {
  console.log(r.name.padEnd(12), (r.total / budget * 100).toFixed(1) + "% of budget");
}