Scheduling আর resource — request, limit, QoS আর autoscaling

apiVersion: v1
kind: Pod
metadata:
  name: api
spec:
  containers:
  - name: app
    image: nginx:1.27-alpine
    imagePullPolicy: IfNotPresent
    resources:
      requests:
        cpu: 250m
        memory: 128Mi
      limits:
        cpu: 500m
        memory: 256Mi

$ kubectl logs cpu-limited
wall=5.00s cpu=0.53s
cpu.max: 10000 100000
cpu.weight: 11
nr_periods 52
nr_throttled 52
throttled_usec 4470241

$ kubectl logs cpu-unlimited
wall=5.00s cpu=5.23s
cpu.max: max 100000

$ kubectl logs mem-hog | tail -3
allocated 16 MiB
allocated 20 MiB
allocated 24 MiB

$ kubectl get pod mem-hog -o jsonpath='{.status.phase} {.status.containerStatuses[0].state.terminated.reason} {.status.containerStatuses[0].state.terminated.exitCode}{"\n"}'
Failed OOMKilled 137

$ kubectl exec qos-burstable -- sh -c 'cat /sys/fs/cgroup/cpu.max /sys/fs/cgroup/cpu.weight /sys/fs/cgroup/memory.max'
20000 100000
11
67108864

$ kubectl get pod too-big
NAME      READY   STATUS    RESTARTS   AGE
too-big   0/1     Pending   0          4s

$ kubectl get events --field-selector involvedObject.name=too-big \
    -o custom-columns=TYPE:.type,REASON:.reason,MSG:.message
TYPE      REASON             MSG
Warning   FailedScheduling   0/1 nodes are available: 1 Insufficient cpu, 1 Insufficient memory. no new claims to deallocate, preemption: 0/1 nodes are available: 1 Preemption is not helpful for scheduling.

$ kubectl get pod -o jsonpath='{range .items[*]}{.metadata.name}{"  "}{.status.qosClass}{"\n"}{end}'
qos-besteffort  BestEffort
qos-burstable  Burstable
qos-guaranteed  Guaranteed

$ kubectl exec qos-guaranteed -- cat /proc/self/oom_score_adj
-997
$ kubectl exec qos-besteffort -- cat /proc/self/oom_score_adj
1000

apiVersion: v1
kind: LimitRange
metadata:
  name: team-defaults
spec:
  limits:
  - type: Container
    defaultRequest: {cpu: 50m, memory: 32Mi}
    default: {cpu: 100m, memory: 64Mi}
    max: {cpu: 500m, memory: 128Mi}
---
apiVersion: v1
kind: ResourceQuota
metadata:
  name: team-quota
spec:
  hard:
    requests.cpu: 300m
    requests.memory: 256Mi
    limits.memory: 512Mi
    pods: "8"

# ১. resources ছাড়া Pod: default বসে গেল
$ kubectl apply -f no-resources.yaml
pod/no-resources created
$ kubectl get pod no-resources -o jsonpath='{.spec.containers[0].resources}{"\n"}{.status.qosClass}{"\n"}'
{"limits":{"cpu":"100m","memory":"64Mi"},"requests":{"cpu":"50m","memory":"32Mi"}}
Burstable

# ২. একটি container limit 2 CPU চাইল
$ kubectl apply -f over-max.yaml
Error from server (Forbidden): … pods "over-max" is forbidden: maximum cpu usage per Container is 500m, but limit is 2

# ৩. একটি Pod quota-র চেয়ে বেশি চাইল
$ kubectl apply -f over-quota.yaml
Error from server (Forbidden): … pods "over-quota" is forbidden: exceeded quota: team-quota, requested: requests.cpu=400m, used: requests.cpu=50m, limited: requests.cpu=300m

# ৪. কতটা খরচ হয়েছে
$ kubectl describe quota team-quota
Name:            team-quota
Namespace:       …
Resource         Used  Hard
--------         ----  ----
limits.memory    64Mi  512Mi
pods             1     8
requests.cpu     50m   300m
requests.memory  32Mi  256Mi

apiVersion: apps/v1
kind: Deployment
metadata:
  name: api
spec:
  replicas: 3
  selector:
    matchLabels:
      app: api
  template:
    metadata:
      labels:
        app: api
    spec:
      nodeSelector:
        kubernetes.io/arch: amd64
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 80
            preference:
              matchExpressions:
              - key: disktype
                operator: In
                values: ["ssd"]
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchLabels:
                app: api
            topologyKey: kubernetes.io/hostname
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: topology.kubernetes.io/zone
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app: api
      tolerations:
      - key: gpu
        operator: Equal
        value: "true"
        effect: NoSchedule
      containers:
      - name: app
        image: nginx:1.27-alpine
        imagePullPolicy: IfNotPresent
        resources:
          requests:
            cpu: 100m
            memory: 64Mi
          limits:
            memory: 128Mi

$ kubectl get pods -o custom-columns=NAME:.metadata.name,STATUS:.status.phase,NODE:.spec.nodeName,PRI:.spec.priority
NAME                         STATUS    NODE             PRI
important                    Running   docker-desktop   100000
skew-web-6c68b9fcc6-jkklz    Running   docker-desktop   0
skew-web-6c68b9fcc6-v7s49    Running   docker-desktop   0
skew-web-6c68b9fcc6-zgdvr    Running   docker-desktop   0
spread-web-d94f67f44-6w29j   Pending   <none>           0
spread-web-d94f67f44-h7ldv   Running   docker-desktop   0
wants-ssd                    Pending   <none>           0

$ kubectl get events --field-selector reason=FailedScheduling -o custom-columns=OBJ:.involvedObject.name,MSG:.message
OBJ                          MSG
spread-web-d94f67f44-6w29j   0/1 nodes are available: 1 node(s) didn't match pod anti-affinity rules. …
wants-ssd                    0/1 nodes are available: 1 node(s) didn't match Pod's node affinity/selector. …

$ kubectl get pod important -o jsonpath='{range .spec.tolerations[*]}{.key}{" "}{.operator}{" "}{.effect}{" "}{.tolerationSeconds}{"\n"}{end}'
node.kubernetes.io/not-ready Exists NoExecute 300
node.kubernetes.io/unreachable Exists NoExecute 300

const NODES = [ // cpu in millicores, mem in MiB, used = sum of requests
  { name: "node-a", cpu: 4000, mem: 8192, usedCpu: 3000, usedMem: 3000, labels: { disk: "ssd" }, taints: [], apps: [] },
  { name: "node-b", cpu: 4000, mem: 8192, usedCpu: 1000, usedMem: 2000, labels: { disk: "hdd" }, taints: [], apps: [] },
  { name: "node-c", cpu: 8000, mem: 16384, usedCpu: 500, usedMem: 1000, labels: { disk: "ssd", gpu: "yes" }, taints: ["gpu=true:NoSchedule"], apps: [] },
];
const PODS = [
  { name: "api-1", app: "api", cpu: 500, mem: 512, antiAffinity: true },
  { name: "api-2", app: "api", cpu: 500, mem: 512, antiAffinity: true },
  { name: "api-3", app: "api", cpu: 500, mem: 512, antiAffinity: true },
  { name: "db", app: "db", cpu: 800, mem: 2048, nodeSelector: { disk: "ssd" } },
  { name: "train", app: "train", cpu: 2000, mem: 4096, nodeSelector: { gpu: "yes" }, tolerates: ["gpu=true:NoSchedule"] },
  { name: "big", app: "big", cpu: 6000, mem: 1024 },
];
function whyNot(n, p) {
  const t = n.taints.find(x => !(p.tolerates || []).includes(x));
  if (t) return `untolerated taint ${t}`;
  const s = Object.entries(p.nodeSelector || {}).find(([k, v]) => n.labels[k] !== v);
  if (s) return `selector ${s[0]}=${s[1]} not matched`;
  if (p.antiAffinity && n.apps.includes(p.app)) return "pod anti-affinity";
  if (n.usedCpu + p.cpu > n.cpu) return `Insufficient cpu (${n.cpu - n.usedCpu}m free)`;
  if (n.usedMem + p.mem > n.mem) return `Insufficient memory (${n.mem - n.usedMem}Mi free)`;
  return "";
}
const score = (n, p) => { // LeastAllocated: more free after placing = higher
  return Math.round(((n.cpu - n.usedCpu - p.cpu) / n.cpu + (n.mem - n.usedMem - p.mem) / n.mem) * 50);
};
for (const p of PODS) {
  const fit = NODES.filter(n => !whyNot(n, p));
  NODES.filter(n => whyNot(n, p)).forEach(n => console.log(`  ${p.name} x ${n.name}: ${whyNot(n, p)}`));
  if (!fit.length) { console.log(`${p.name}: Pending (0/${NODES.length} nodes available)`); continue; }
  const ranked = fit.map(n => [score(n, p), n]).sort((a, b) => b[0] - a[0]);
  const n = ranked[0][1];
  n.usedCpu += p.cpu; n.usedMem += p.mem; n.apps.push(p.app);
  console.log(`${p.name}: ${n.name}  (scores: ${ranked.map(r => r[1].name + "=" + r[0]).join(" ")})`);
}

// Topology spread across zones: without and with maxSkew. Edit and run.
const NODES = [["a1", "a"], ["a2", "a"], ["b1", "b"], ["c1", "c"]];
const REPLICAS = 6, MAX_SKEW = 1, LOST_ZONE = "a";

function place(useSpread) {
  const pods = Object.fromEntries(NODES.map(([n]) => [n, 0]));
  const inZone = z => NODES.filter(([, y]) => y === z).reduce((s, [n]) => s + pods[n], 0);
  const zones = [...new Set(NODES.map(([, z]) => z))];
  for (let r = 0; r < REPLICAS; r++) {
    const low = Math.min(...zones.map(inZone));
    const ok = NODES.filter(([, z]) => !useSpread || inZone(z) + 1 - low <= MAX_SKEW);
    const pick = ok.reduce((b, c) => (pods[c[0]] < pods[b[0]] ? c : b)); // fewest Pods
    pods[pick[0]]++;
  }
  const line = zones.map(z => `zone-${z}=${inZone(z)}`).join(" ");
  const left = REPLICAS - inZone(LOST_ZONE);
  console.log(`${useSpread ? "with spread   " : "without spread"}  ${line}  lose zone-${LOST_ZONE}: ${left} of ${REPLICAS} left`);
}
place(false);
place(true);

apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: abcl-kh-payment
value: 100000
preemptionPolicy: PreemptLowerPriority
globalDefault: false
description: "Payment workloads."

$ kubectl get pod important -o jsonpath='{.spec.priorityClassName} {.spec.priority} {.spec.preemptionPolicy}{"\n"}'
abcl-kh-high 100000 PreemptLowerPriority

$ kubectl get priorityclass
NAME                      VALUE        GLOBAL-DEFAULT   AGE     PREEMPTIONPOLICY
system-cluster-critical   2000000000   false            …       PreemptLowerPriority
system-node-critical      2000001000   false            …       PreemptLowerPriority

$ helm install abcl-kh-metrics metrics-server \
    --repo https://kubernetes-sigs.github.io/metrics-server/ \
    -n … --set 'args={--kubelet-insecure-tls}' \
    --set resources.requests.cpu=50m --set resources.requests.memory=64Mi \
    --set resources.limits.memory=128Mi
NAME: abcl-kh-metrics
…
STATUS: deployed
REVISION: 1
…
$ kubectl top pods
NAME                                              CPU(cores)   MEMORY(bytes)
abcl-kh-metrics-metrics-server-7b67f97c78-kwqtg   4m           18Mi
web-675bfcb8b7-tsp9t                              1m           22Mi

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: web
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: web
  minReplicas: 1
  maxReplicas: 4
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 50
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 60

# t = লোড Pod তৈরির পরের সেকেন্ড (TARGETS, REPLICAS)
t=15s    cpu: 62%/50%    1
t=75s    cpu: 233%/50%   2
t=135s   cpu: 188%/50%   4

$ kubectl describe hpa web
…
  resource cpu on pods  (as a percentage of request):  188% (94m) / 50%
Behavior:
  Scale Up:
    Stabilization Window: 0 seconds
    Select Policy: Max
    Policies:
      - Type: Pods     Value: 4    Period: 15 seconds
      - Type: Percent  Value: 100  Period: 15 seconds
  Scale Down:
    Stabilization Window: 60 seconds
…
  ScalingLimited  True    TooManyReplicas   the desired replica count is more than the maximum replica count

$ kubectl get events --field-selector involvedObject.kind=HorizontalPodAutoscaler \
    -o custom-columns=REASON:.reason,COUNT:.count,MSG:.message
REASON              COUNT   MSG
SuccessfulRescale   1       New size: 2; reason: cpu resource utilization (percentage of request) above target
SuccessfulRescale   1       New size: 4; reason: cpu resource utilization (percentage of request) above target
SuccessfulRescale   1       New size: 1; reason: All metrics below target

// HPA loop with tolerance, scale-up policy and scale-down window.
const CFG = {
  request: 200, target: 60,  // millicores per Pod, percent of request
  min: 2, max: 10,
  tolerance: 0.10,           // ignore a ratio inside 1 +/- this
  downWindow: 4,             // intervals (15 s each) of memory for scale-down
};
// Total CPU the app needs in each 15 s interval, in millicores.
const LOAD = [230, 250, 900, 1800, 2000, 2000, 1400, 500, 300, 300, 300, 300, 300, 300];

let replicas = 2;
const history = [];
console.log(" t  load  pods  util  ratio  raw  next");
LOAD.forEach((load, i) => {
  const util = load / replicas / CFG.request * 100;
  const ratio = util / CFG.target;
  let raw = replicas;
  if (Math.abs(ratio - 1) > CFG.tolerance) raw = Math.ceil(replicas * ratio);
  raw = Math.min(CFG.max, Math.max(CFG.min, raw));
  history.push(raw);
  if (history.length > CFG.downWindow) history.shift();
  let next = raw;
  if (raw < replicas) next = Math.min(replicas, Math.max(...history)); // window keeps the highest
  if (raw > replicas) next = Math.min(raw, Math.max(replicas * 2, replicas + 4)); // 100% or 4 Pods
  const row = [String(i * 15).padStart(3), String(load).padStart(5), String(replicas).padStart(4),
    (util.toFixed(0) + "%").padStart(5), ratio.toFixed(2).padStart(6), String(raw).padStart(4), String(next).padStart(4)];
  console.log(row.join(" "));
  replicas = next;
});

// Requests vs usage waste report. Prices are ASSUMED round numbers.
const PRICE_CPU = 0.04;   // $ per vCPU-hour (assumed)
const PRICE_GIB = 0.005;  // $ per GiB-hour (assumed)
const HOURS = 720;        // 30-day month
const HEADROOM = 1.3;     // keep 30% above p95
const APPS = [ // cpu in millicores, mem in MiB, p95 = measured over two weeks
  { name: "web",    pods: 6, cpu: 1000, p95Cpu: 180, mem: 1024, p95Mem: 300 },
  { name: "api",    pods: 4, cpu: 500,  p95Cpu: 320, mem: 512,  p95Mem: 400 },
  { name: "worker", pods: 3, cpu: 250,  p95Cpu: 240, mem: 256,  p95Mem: 310 },
  { name: "report", pods: 2, cpu: 2000, p95Cpu: 150, mem: 4096, p95Mem: 900 },
];
const up = (x, step) => Math.ceil(x / step) * step;
const cost = (m, mi, pods) =>
  (m / 1000 * PRICE_CPU + mi / 1024 * PRICE_GIB) * pods * HOURS;

let nowTotal = 0, newTotal = 0;
console.log("app      cpu req>new  mem req>new  $/mo now>new  note");
for (const a of APPS) {
  const newCpu = up(a.p95Cpu * HEADROOM, 10), newMem = up(a.p95Mem * HEADROOM, 16);
  const now = cost(a.cpu, a.mem, a.pods), next = cost(newCpu, newMem, a.pods);
  nowTotal += now; newTotal += next;
  const note = a.p95Mem > a.mem ? "RISK: usage above memory request"
    : a.p95Cpu > 0.9 * a.cpu ? "RISK: CPU near request"
    : a.cpu > 2 * newCpu ? "WASTE" : "ok";
  console.log(a.name.padEnd(8), `${a.cpu}>${newCpu}`.padEnd(10), `${a.mem}>${newMem}`.padEnd(12),
    `${now.toFixed(0)}>${next.toFixed(0)}`.padEnd(13), note);
}
const saved = nowTotal - newTotal;
console.log(`\ntotal now $${nowTotal.toFixed(0)}/mo, after rightsizing $${newTotal.toFixed(0)}/mo`);
console.log(`saving $${saved.toFixed(0)}/mo (${(saved / nowTotal * 100).toFixed(0)}%)`);