ক্যাপস্টোন — code থেকে production: container, Kubernetes, Terraform, pipeline আর observability

VERSION = os.environ.get("BUILD_SHA", "dev")      # which commit am I?
hits = {("/", "200"): 0, ("/", "500"): 0}         # series exist from the first scrape

class Handler(BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path == "/healthz":
            return self.send(503 if draining else 200,
                             "draining\n" if draining else "ok\n", "text/plain")
        if self.path == "/metrics":
            return self.send(200, metrics(), "text/plain; version=0.0.4")
        ...

def on_term(signum, frame):
    global draining
    draining = True          # /healthz now says 503: stop sending me traffic
    threading.Thread(target=drain, args=(srv,), daemon=True).start()

# Stage 1: run the tests. If they fail, nothing after this can be built.
FROM python:3.12-alpine AS test
WORKDIR /src
COPY app.py test_app.py ./
RUN python -m unittest -v

# Stage 2: the image we ship. It takes app.py FROM the test stage.
FROM python:3.12-alpine AS runtime
ARG BUILD_SHA=dev
ENV BUILD_SHA=$BUILD_SHA PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1
WORKDIR /app
COPY --from=test /src/app.py ./app.py
USER 10001:10001
EXPOSE 8000
CMD ["python", "app.py"]

$ docker run --rm --hostname demo --entrypoint sh $IMAGE -c 'echo uid=$(id -u); ls -la /app'
uid=10001
total 12
…
-rwxr-xr-x    1 root     root          3424 Oct 31 10:04 app.py

#!/usr/bin/env bash
# A local pipeline: the same stages a hosted CI would run on every push.
set -euo pipefail
REGISTRY=127.0.0.1:18761
IMAGE=abcl-cap-boighor

[ -z "$(git status --porcelain)" ] || { echo "STOP: uncommitted changes"; exit 1; }
SHA=$(git rev-parse --short=12 HEAD)
REF="$REGISTRY/$IMAGE:$SHA"

echo "== 1/4 test   commit $SHA"
docker build --quiet --target test --output type=cacheonly .
echo "unit tests passed"

echo "== 2/4 build  $REF"
docker build --quiet --build-arg BUILD_SHA="$SHA" -t "$REF" . >/dev/null
docker image ls "$REF" --format 'size {{.Size}}'

echo "== 3/4 smoke  run the image, then ask it for /healthz and /"
docker run -d --name abcl-cap-smoke --hostname demo \
  -p 127.0.0.1:18763:8000 "$REF" >/dev/null
trap 'docker rm -fv abcl-cap-smoke >/dev/null' EXIT
for _ in 1 2 3 4 5 6 7 8 9 10; do
  curl -fsS http://127.0.0.1:18763/healthz >/dev/null 2>&1 && break
  sleep 0.5
done
curl -fsS http://127.0.0.1:18763/healthz
curl -fsS http://127.0.0.1:18763/

echo "== 4/4 push   only a tested image reaches the registry"
docker push "$REF" | tail -n 1
echo "ready to deploy: image_tag=$SHA"

$ bash pipeline.sh
== 1/4 test   commit bc5cc517cd8c
unit tests passed
== 2/4 build  127.0.0.1:18761/abcl-cap-boighor:bc5cc517cd8c
size 74.1MB
== 3/4 smoke  run the image, then ask it for /healthz and /
ok
{"shop": "boighor", "version": "bc5cc517cd8c", "greeting": "Welcome to Boighor", "book": "Padma Nadir Majhi"}
== 4/4 push   only a tested image reaches the registry
bc5cc517cd8c: digest: sha256:94502e31e84e33ef892cf7c8eba368e535014527734a79749d6d8b2d47f0e260 size: 856
ready to deploy: image_tag=bc5cc517cd8c

name: boighor-ci
on:
  push:
    branches: [main]
permissions:
  contents: read
  packages: write
jobs:
  deliver:
    runs-on: ubuntu-24.04
    env:
      IMAGE: ghcr.io/example-org/boighor
    steps:
      - uses: actions/checkout@v4
      - name: Test
        run: docker build --target test --output type=cacheonly .
      - name: Build
        run: docker build --build-arg BUILD_SHA=${{ github.sha }} -t "$IMAGE:${{ github.sha }}" .
      - name: Smoke
        run: |
          docker run -d --name smoke -p 127.0.0.1:8000:8000 "$IMAGE:${{ github.sha }}"
          for i in 1 2 3 4 5 6 7 8 9 10; do curl -fsS http://127.0.0.1:8000/healthz && break; sleep 1; done
          curl -fsS http://127.0.0.1:8000/
      - name: Log in to the registry
        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
      - name: Push
        run: docker push "$IMAGE:${{ github.sha }}"

variable "image_tag" {
  type        = string
  description = "Commit SHA that the pipeline pushed. Never 'latest'."
}

locals {
  ns     = "abcl-cap"
  labels = { app = "boighor" }
  image  = "127.0.0.1:18761/abcl-cap-boighor:${var.image_tag}"
}

resource "kubernetes_deployment_v1" "shop" {
  metadata {
    name      = "boighor"
    namespace = kubernetes_namespace_v1.shop.metadata[0].name
    labels    = local.labels
  }
  spec {
    replicas                  = var.replicas
    min_ready_seconds         = 3
    progress_deadline_seconds = 45
    selector {
      match_labels = local.labels
    }
    strategy {
      type = "RollingUpdate"
      rolling_update {
        max_surge       = "1"
        max_unavailable = "0"
      }
    }
    template {
      metadata {
        labels = local.labels
      }
      spec {
        termination_grace_period_seconds = 10
        security_context {
          run_as_non_root = true
          seccomp_profile {
            type = "RuntimeDefault"
          }
        }
        container {
          name  = "boighor"
          image = local.image
          port {
            name           = "http"
            container_port = 8000
          }
          env_from {
            config_map_ref {
              name = kubernetes_config_map_v1.shop.metadata[0].name
            }
          }
          readiness_probe {
            http_get {
              path = "/healthz"
              port = "http"
            }
            period_seconds    = 2
            failure_threshold = 2
          }
          liveness_probe {
            http_get {
              path = "/healthz"
              port = "http"
            }
            period_seconds    = 10
            failure_threshold = 3
          }
          resources {
            requests = { cpu = "20m", memory = "32Mi" }
            limits   = { cpu = "200m", memory = "64Mi" }
          }
          security_context {
            allow_privilege_escalation = false
            read_only_root_filesystem  = true
            capabilities {
              drop = ["ALL"]
            }
          }
        }
      }
    }
  }
  lifecycle {
    ignore_changes = [spec[0].replicas] # the HPA owns this number
  }
}

$ terraform plan -var image_tag=bc5cc517cd8c -out=v1.plan
…
Plan: 5 to add, 0 to change, 0 to destroy.

$ terraform apply v1.plan
kubernetes_namespace_v1.shop: Creating...
kubernetes_namespace_v1.shop: Creation complete after 0s [id=abcl-cap]
…
kubernetes_config_map_v1.shop: Creation complete after 0s [id=abcl-cap/boighor-config]
kubernetes_service_v1.shop: Creation complete after 0s [id=abcl-cap/boighor]
kubernetes_deployment_v1.shop: Creation complete after 8s [id=abcl-cap/boighor]
…
kubernetes_horizontal_pod_autoscaler_v2.shop: Creation complete after 0s [id=abcl-cap/boighor]

Apply complete! Resources: 5 added, 0 changed, 0 destroyed.

$ kubectl -n abcl-cap get deploy,rs,pod,svc,hpa,cm    # অংশবিশেষ
NAME                      READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/boighor   3/3     3            3           8s

NAME                          READY   STATUS    RESTARTS   AGE
pod/boighor-9dc6dbc5d-2rrpz   1/1     Running   0          8s
pod/boighor-9dc6dbc5d-fvqfz   1/1     Running   0          8s
pod/boighor-9dc6dbc5d-xl7r9   1/1     Running   0          8s

NAME              TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE
service/boighor   ClusterIP   10.105.172.42   <none>        8000/TCP   8s

NAME                                          REFERENCE            TARGETS              MINPODS   MAXPODS   …
horizontalpodautoscaler.autoscaling/boighor   Deployment/boighor   cpu: <unknown>/60%   3         5         …

$ kubectl -n abcl-cap get pod -l app=boighor    # rollout-এর মাঝখানে
boighor-9dc6dbc5d-2rrpz   1/1   Terminating         0     74s
boighor-9dc6dbc5d-fvqfz   1/1   Running             0     74s
boighor-9dc6dbc5d-xl7r9   1/1   Running             0     74s
boighor-9fdb47867-gjddn   1/1   Running             0     5s
boighor-9fdb47867-xhhrh   0/1   ContainerCreating   0     0s

$ kubectl -n abcl-cap logs loadgen
…
t=  64s ok=604 http5xx=0 dropped=0 b538f57:418 bc5cc51:186

$ terraform apply -auto-approve -var image_tag=1002160772e5
kubernetes_deployment_v1.shop: Modifying... [id=abcl-cap/boighor]
…
Error: Deployment exceeded its progress deadline

  with kubernetes_deployment_v1.shop,
  on main.tf line 49, in resource "kubernetes_deployment_v1" "shop":
…
$ kubectl -n abcl-cap get pod -l app=boighor
NAME                       READY   STATUS    RESTARTS      AGE
boighor-845c76dc89-hgpn6   0/1     Running   1 (21s ago)   55s
boighor-9fdb47867-7x9zp    1/1     Running   0             105s
boighor-9fdb47867-pgn9z    1/1     Running   0             116s
boighor-9fdb47867-vkmjp    1/1     Running   0             110s

$ kubectl -n abcl-cap logs boighor-845c76dc89-hgpn6
catalog cache failed: [Errno 30] Read-only file system: '/tmp/boighor-catalog.json'
boighor 1002160772e5 listening on :8000

$ kubectl -n abcl-cap logs loadgen --tail=2
t= 215s ok=100 http5xx=0 dropped=0 b538f57:100
t= 220s ok=100 http5xx=0 dropped=0 b538f57:100

$ terraform apply -auto-approve -var image_tag=b538f571b047
kubernetes_deployment_v1.shop: Modifying... [id=abcl-cap/boighor]
kubernetes_deployment_v1.shop: Modifications complete after 1s [id=abcl-cap/boighor]

Apply complete! Resources: 0 added, 1 changed, 0 destroyed.

global:
  scrape_interval: 5s
  evaluation_interval: 5s
rule_files:
  - /etc/prometheus/slo.rules.yml
scrape_configs:
  - job_name: boighor
    kubernetes_sd_configs:
      - role: pod
        namespaces:
          names: [abcl-cap]
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_app]
        regex: boighor
        action: keep
      - source_labels: [__meta_kubernetes_pod_name]
        target_label: pod

groups:
  - name: boighor-slo
    rules:
      - record: boighor:error_ratio:rate5m
        expr: |
          sum(rate(boighor_http_requests_total{path="/",code=~"5.."}[5m]))
          / sum(rate(boighor_http_requests_total{path="/"}[5m]))
      - record: boighor:error_ratio:rate1h
        expr: |
          sum(rate(boighor_http_requests_total{path="/",code=~"5.."}[1h]))
          / sum(rate(boighor_http_requests_total{path="/"}[1h]))
      - alert: BoighorFastBurn
        expr: |
          boighor:error_ratio:rate5m > (14.4 * 0.001)
          and boighor:error_ratio:rate1h > (14.4 * 0.001)
        for: 1m
        labels:
          severity: page
        annotations:
          summary: "Boighor is burning its 30-day error budget 14.4x too fast"

$ kubectl -n abcl-cap exec $POD -- python -c "import urllib.request as u; print(u.urlopen('http://127.0.0.1:8000/metrics').read().decode())"
boighor_build_info{version="a12de41eaff0"} 1
boighor_http_requests_total{path="/",code="200"} 168
boighor_http_requests_total{path="/",code="500"} 0
boighor_request_seconds_bucket{le="0.005"} 167
…
boighor_request_seconds_count 168

$ python q.py 'sum(rate(boighor_http_requests_total{path="/"}[1m]))'
19.3984808456819
$ python q.py 'boighor:error_ratio:rate5m'
0
$ python q.py --alerts
no alerts

$ kubectl -n abcl-cap delete pod boighor-9fdb47867-7x9zp
pod "boighor-9fdb47867-7x9zp" deleted from abcl-cap namespace

$ kubectl -n abcl-cap logs loadgen --tail=3
t= 265s ok=100 http5xx=0 dropped=0 b538f57:100
t= 270s ok=100 http5xx=0 dropped=0 b538f57:100
t= 275s ok=100 http5xx=0 dropped=0 b538f57:100

$ kubectl -n abcl-cap delete pod boighor-9fdb47867-c42j6 --grace-period=0 --force
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. …
pod "boighor-9fdb47867-c42j6" force deleted from abcl-cap namespace

$ kubectl -n abcl-cap logs loadgen --tail=2
t= 290s ok=100 http5xx=0 dropped=0 b538f57:100
t= 295s ok=100 http5xx=0 dropped=0 b538f57:100

$ kubectl -n abcl-cap get pod -l app=boighor
NAME                      READY   STATUS    RESTARTS   AGE
boighor-9fdb47867-pgn9z   1/1     Running   0          3m10s
boighor-9fdb47867-q7bjm   1/1     Running   0          20s
boighor-9fdb47867-vkmjp   1/1     Running   0          3m4s

$ kubectl -n abcl-cap logs loadgen    # rollout শেষে, প্রতি ৫ সেকেন্ড
t= 355s ok=75 http5xx=25 dropped=0 f9134a4:75
t= 360s ok=76 http5xx=24 dropped=0 f9134a4:76

$ for i in 1 2 3 4; do curl -s -o - -w ' [%{http_code}]\n' http://127.0.0.1:18764/ | tr -d '\n'; echo; done
{"shop": "boighor", …, "book": "Padma Nadir Majhi", "price_bdt": 300, "rating": 4.7} [200]
{"shop": "boighor", …, "book": "Lalsalu", "price_bdt": 220, "rating": 5.0} [200]
{"shop": "boighor", …, "book": "Hajar Bachhar Dhore", "price_bdt": 180, "rating": 4.0} [200]
{"error": "ZeroDivisionError"} [500]

$ docker logs abcl-cap-dbg
boighor f9134a436006 listening on :8000

$ bash pipeline.sh; echo pipeline_exit=$?
== 1/4 test   commit c60cc618efdf
…
test_home_works_for_every_book (test_app.HomeTest.test_home_works_for_every_book) ... ERROR
…
ZeroDivisionError: division by zero
…
FAILED (errors=1)
…
ERROR: failed to build: failed to solve: process "/bin/sh -c python -m unittest -v" did not complete …
pipeline_exit=1

// Request-budget tracker for the Boighor SLO. Change the numbers and run again.
const SLO = 0.999, REQ_PER_SEC = 20, DAYS = 30;
const monthly = REQ_PER_SEC * 86400 * DAYS;
const budget = Math.round(monthly * (1 - SLO));
console.log(`requests in ${DAYS} days: ${monthly.toLocaleString("en-US")}`);
console.log(`failed requests allowed: ${budget.toLocaleString("en-US")}`);

// Events of the month: [name, failed requests]. The first three are from the real run.
const events = [
  ["v2 rolling update", 0],
  ["bad image stopped by readiness", 0],
  ["rating bug, 3 minutes", 811],
  ["(your event here)", 0],
];
let used = 0;
for (const [name, failed] of events) {
  used += failed;
  const left = 100 * (budget - used) / budget;
  console.log(name.padEnd(32), String(failed).padStart(6), `${left.toFixed(2)}% left`.padStart(13));
}

// How fast would a given error ratio finish the budget?
function burn(errorRatio) {
  const rate = errorRatio / (1 - SLO);
  const hours = DAYS * 24 / rate;
  return { rate, hours };
}
for (const e of [0.002, 0.0144, 0.25, 1]) {
  const { rate, hours } = burn(e);
  console.log(`error ${(e * 100).toFixed(2).padStart(6)}%  burn ${rate.toFixed(1).padStart(6)}x  budget gone in ${hours.toFixed(1).padStart(6)} h`);
}
// If nobody had rolled back: the same bug through an 8-hour night.
const night = REQ_PER_SEC * 8 * 3600 * 0.25;
console.log(`8 h at 25% errors: ${night} failed, ${(100 * night / budget).toFixed(0)}% of the budget`);

// Monthly cost model of the Boighor design. ALL PRICES ARE ASSUMED ROUND NUMBERS.
// They are not any provider's real price: check the provider's pricing page.
const HOURS = 720;                       // 30-day month
const P = { controlPlane: 0.10, node: 0.04, loadBalancer: 0.025, nat: 0.05,
            egressGB: 0.10, natGB: 0.05, logGB: 0.50, registryGB: 0.10 };
const USE = { egressGB: 200, natGB: 20, logGB: 15, registryGB: 5, millionReq: 51.84 };

function design(name, nodes, nodePrice, hasControlPlane, hasNat) {
  const items = {
    "control plane": hasControlPlane ? P.controlPlane * HOURS : 0,
    "worker nodes": nodes * nodePrice * HOURS,
    "load balancer": P.loadBalancer * HOURS,
    "NAT gateway": hasNat ? P.nat * HOURS + P.natGB * USE.natGB : 0,
    "internet egress": P.egressGB * USE.egressGB,
    "logs and metrics": P.logGB * USE.logGB,
    "image registry": P.registryGB * USE.registryGB,
  };
  const total = Object.values(items).reduce((a, b) => a + b, 0);
  return { name, items, total };
}
const designs = [
  design("A managed K8s, 3 nodes", 3, P.node, true, true),
  design("B same, 1-yr commitment -30%", 3, P.node * 0.7, true, true),
  design("C two VMs, no K8s, no NAT", 2, P.node, false, false),
];
for (const d of designs) {
  console.log(d.name);
  for (const [k, v] of Object.entries(d.items))
    if (v > 0) console.log("  " + k.padEnd(18), ("$" + v.toFixed(2)).padStart(9));
  console.log("  " + "TOTAL / month".padEnd(18), ("$" + d.total.toFixed(2)).padStart(9),
    `  $${(d.total / USE.millionReq).toFixed(2)} per million requests`);
}

$ kubectl -n abcl-cap delete pod loadgen
$ kubectl -n abcl-cap delete -f prometheus.yaml
$ kubectl -n abcl-cap delete configmap loadgen-script

$ terraform destroy -auto-approve -var image_tag=a12de41eaff0
kubernetes_horizontal_pod_autoscaler_v2.shop: Destruction complete after 0s
kubernetes_service_v1.shop: Destruction complete after 0s
kubernetes_deployment_v1.shop: Destruction complete after 0s
kubernetes_config_map_v1.shop: Destruction complete after 0s
kubernetes_namespace_v1.shop: Destruction complete after 7s

Destroy complete! Resources: 5 destroyed.

$ kubectl get ns abcl-cap
Error from server (NotFound): namespaces "abcl-cap" not found

$ docker rm -fv abcl-cap-registry      # -v: এর নামহীন volume-ও যাবে
abcl-cap-registry
$ docker rmi prom/prometheus:v3.5.0 registry:2
…
$ docker ps -a --format '{{.Names}}' | grep -c abcl-cap
0

// Production readiness score for Boighor. Flip done to true as you fix each item.
// Each row: [area, check, weight, launch blocker?, done?]
const checks = [
  ["delivery", "image tagged by commit SHA",        3, true,  true],
  ["delivery", "tests can stop the build",          3, true,  true],
  ["delivery", "app resources in Terraform",        3, true,  true],
  ["delivery", "remote state with locking",         3, true,  false],
  ["runtime",  "non-root, read-only filesystem",    3, true,  true],
  ["runtime",  "readiness and graceful shutdown",   3, true,  true],
  ["runtime",  "HPA has a working metric source",   2, false, false],
  ["runtime",  "PodDisruptionBudget",               1, false, false],
  ["observe",  "SLO with a burn-rate rule",         3, true,  true],
  ["observe",  "alert reaches a real person",       3, true,  false],
  ["observe",  "app logs every 5xx with a cause",   2, false, false],
  ["safety",   "TLS at the public edge",            3, true,  false],
  ["safety",   "rollback rehearsed",                3, true,  true],
  ["safety",   "chaos test passed",                 2, false, true],
  ["money",    "budget alert and cost tags",        2, false, false],
];
const areas = {};
let got = 0, all = 0;
for (const [area, , weight, , done] of checks) {
  areas[area] ??= { got: 0, all: 0 };
  areas[area].all += weight; all += weight;
  if (done) { areas[area].got += weight; got += weight; }
}
for (const [name, a] of Object.entries(areas))
  console.log(name.padEnd(9), `${a.got}/${a.all}`.padStart(6), "#".repeat(a.got).padEnd(13, "."));
const score = (100 * got / all).toFixed(0);
const blockers = checks.filter(c => c[3] && !c[4]).map(c => c[1]);
console.log(`score ${got}/${all} = ${score}%`);
console.log(blockers.length ? `NOT READY, ${blockers.length} blockers:` : "READY to launch");
blockers.forEach(b => console.log("  - " + b));