Kubernetes নিরাপত্তা — RBAC, NetworkPolicy আর Pod Security

# kubeconfig with a fake token
$ kubectl --kubeconfig bad.kubeconfig get pods -n abcl-kx
error: You must be logged in to the server (Unauthorized)

# valid identity, no permission
$ kubectl get secrets -n abcl-kx \
    --as=system:serviceaccount:abcl-kx:report-app
Error from server (Forbidden): secrets is forbidden: User
"system:serviceaccount:abcl-kx:report-app" cannot list resource "secrets"
in API group "" in the namespace "abcl-kx"

$ kubectl get pod -n kube-system -l component=kube-apiserver -o yaml \
    | grep -E -e '--authorization-mode|--enable-admission'
      - --authorization-mode=Node,RBAC
      - --enable-admission-plugins=NodeRestriction

apiVersion: v1
kind: ServiceAccount
metadata:
  name: report-app
  namespace: abcl-kx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: pod-reader
  namespace: abcl-kx
rules:
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: report-app-pod-reader
  namespace: abcl-kx
subjects:
  - kind: ServiceAccount
    name: report-app
    namespace: abcl-kx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: pod-reader

$ SA=system:serviceaccount:abcl-kx:report-app
$ kubectl auth can-i list pods -n abcl-kx --as=$SA
yes
$ kubectl auth can-i delete pods -n abcl-kx --as=$SA
no
$ kubectl auth can-i get secrets -n abcl-kx --as=$SA
no
$ kubectl auth can-i list pods -n default --as=$SA
no
$ kubectl auth can-i --list -n abcl-kx --as=$SA
Resources   Non-Resource URLs   Resource Names   Verbs
…
pods        []                  []               [get list watch]
…

# AUD=system:serviceaccount:abcl-kx:auditor
# auditor has ClusterRole "view" through a RoleBinding in abcl-kx
$ kubectl auth can-i get deployments.apps -n abcl-kx --as=$AUD
yes
$ kubectl auth can-i get secrets -n abcl-kx --as=$AUD
no
$ kubectl auth can-i get pods -n default --as=$AUD
no

# a ClusterRoleBinding adds one cluster-scoped permission
$ kubectl auth can-i list namespaces --as=$AUD
no
$ kubectl apply -f 12-crb.yaml
clusterrole.rbac.authorization.k8s.io/abcl-kx-ns-reader created
clusterrolebinding.rbac.authorization.k8s.io/abcl-kx-ns-reader created
$ kubectl auth can-i list namespaces --as=$AUD
Warning: resource 'namespaces' is not namespace scoped

yes

// RBAC evaluator: rules only add permission. Edit BINDINGS or ASK and run again.
const ROLES = {
  'Role/pod-reader':     [{ res: ['pods'], verbs: ['get', 'list', 'watch'] }],
  'ClusterRole/view':    [{ res: ['pods', 'services'], verbs: ['get', 'list', 'watch'] }],
  'ClusterRole/ns-read': [{ res: ['namespaces'], verbs: ['get', 'list'] }],
  'Role/do-everything':  [{ res: ['*'], verbs: ['*'] }],
};
const BINDINGS = [
  { kind: 'RoleBinding', ns: 'shop', who: 'report-app', ref: 'Role/pod-reader' },
  { kind: 'RoleBinding', ns: 'shop', who: 'auditor', ref: 'ClusterRole/view' },
  { kind: 'ClusterRoleBinding', who: 'auditor', ref: 'ClusterRole/ns-read' },
  { kind: 'RoleBinding', ns: 'shop', who: 'lazy-app', ref: 'Role/do-everything' },
];
const ASK = [
  ['report-app', 'list', 'pods', 'shop'],
  ['report-app', 'delete', 'pods', 'shop'],
  ['report-app', 'list', 'pods', 'bank'],
  ['auditor', 'get', 'secrets', 'shop'],
  ['auditor', 'list', 'namespaces', '(cluster)'],
  ['lazy-app', 'get', 'secrets', 'shop'],
  ['lazy-app', 'get', 'secrets', 'bank'],
];
const hit = (list, x) => list.includes('*') || list.includes(x);
function check(who, verb, res, ns) {
  for (const b of BINDINGS) {
    if (b.who !== who) continue;
    if (b.kind === 'RoleBinding' && b.ns !== ns) continue; // scope of a RoleBinding
    const ok = ROLES[b.ref].some(r => hit(r.res, res) && hit(r.verbs, verb));
    if (ok) return 'yes  via ' + b.kind + ' -> ' + b.ref;
  }
  return 'no   nothing matched, so denied';
}
for (const [who, verb, res, ns] of ASK) {
  console.log((who + ' ' + verb + ' ' + res + ' in ' + ns).padEnd(38), check(who, verb, res, ns));
}

# LAZY=system:serviceaccount:abcl-kx:lazy-app
# Role do-everything: apiGroups ["*"], resources ["*"], verbs ["*"]
$ kubectl auth can-i get secrets -n abcl-kx --as=$LAZY
yes
$ kubectl auth can-i create rolebindings.rbac.authorization.k8s.io \
    -n abcl-kx --as=$LAZY
yes
$ kubectl auth can-i create pods/exec -n abcl-kx --as=$LAZY
yes
$ kubectl auth can-i get secrets -n default --as=$LAZY
no
$ kubectl auth can-i --list -n abcl-kx --as=$LAZY | head -2
Resources   Non-Resource URLs   Resource Names   Verbs
*.*         []                  []               [*]

$ kubectl create -f 15-try.yaml --dry-run=server --as=$AUD
Error from server (Forbidden): error when creating "15-try.yaml":
roles.rbac.authorization.k8s.io "secret-reader-try" is forbidden: user
"system:serviceaccount:abcl-kx:auditor" (groups=[…]) is attempting to
grant RBAC permissions not currently held:
{APIGroups:[""], Resources:["secrets"], Verbs:["get"]}

# projected volume on the Pod (added by admission)
$ kubectl get pod hardened -n abcl-kx -o json
…
  "name": "kube-api-access-x8rks"
  "serviceAccountToken": { "expirationSeconds": 3607, "path": "token" }
…
  "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
  "readOnly": true

# claims decoded inside the Pod with a small Python script.
# Only claims are printed, never the token itself.
lifetime_seconds 31536000
aud ['https://kubernetes.default.svc.cluster.local']
sub system:serviceaccount:abcl-kx:report-app
pod hardened warnafter 3607

$ kubectl create token report-app -n abcl-kx --duration=10m \
    --bound-object-kind=Pod --bound-object-name=hardened \
    --bound-object-uid=…          # stored, never printed
$ kubectl --kubeconfig t.kubeconfig get pods -n abcl-kx
NAME       READY   STATUS    RESTARTS   AGE
hardened   1/1     Running   0          2s
$ kubectl delete pod hardened -n abcl-kx --grace-period=1
pod "hardened" deleted from abcl-kx namespace

# same token, right after the delete and about 4 seconds later
No resources found in abcl-kx namespace.
No resources found in abcl-kx namespace.
# same token, about 12 seconds later
error: You must be logged in to the server (Unauthorized)

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-ingress
  namespace: abcl-kx
spec:
  podSelector: {}
  policyTypes: ["Ingress"]

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-frontend-to-web
  namespace: abcl-kx
spec:
  podSelector:
    matchLabels:
      app: web
  policyTypes: ["Ingress"]
  ingress:
    - from:
        - podSelector:
            matchLabels:
              role: frontend
      ports:
        - protocol: TCP
          port: 80

# each result line: kubectl exec POD -- wget -q -O /dev/null -T 3 http://web
# exit 0 means the request succeeded
# before any policy
frontend -> web: exit=0
guest    -> web: exit=0

$ kubectl apply -f 21-deny.yaml
networkpolicy.networking.k8s.io/default-deny-ingress created
frontend -> web: exit=0
guest    -> web: exit=0

$ kubectl apply -f 22-allow.yaml
networkpolicy.networking.k8s.io/allow-frontend-to-web created
frontend -> web: exit=0
guest    -> web: exit=0

$ kubectl get netpol -n abcl-kx
NAME                    POD-SELECTOR   AGE
allow-frontend-to-web   app=web        1s
default-deny-ingress    <none>         5s

$ kubectl exec -n abcl-kx guest -- wget -q -O /dev/null -T 3 http://web
$ echo "exit code: $?"
exit code: 0

// NetworkPolicy reachability matrix (ingress only). Edit POLICIES and run again.
const PODS = {
  frontend: { role: 'frontend' }, guest: { role: 'guest' },
  web: { app: 'web' }, db: { app: 'db' },
};
const POLICIES = [
  { name: 'default-deny', select: {}, from: [] },
  { name: 'frontend-to-web', select: { app: 'web' }, from: [{ role: 'frontend' }] },
  { name: 'web-to-db', select: { app: 'db' }, from: [{ app: 'web' }] },
];
// An empty selector {} matches every Pod.
const matches = (sel, labels) => Object.keys(sel).every(k => labels[k] === sel[k]);

function allowed(src, dst, enforced) {
  if (!enforced) return true;                // the plugin ignores every policy
  const mine = POLICIES.filter(p => matches(p.select, PODS[dst]));
  if (mine.length === 0) return true;        // not selected: not isolated
  return mine.some(p => p.from.some(f => matches(f, PODS[src]))); // union
}
const names = Object.keys(PODS);
for (const enforced of [true, false]) {
  console.log(enforced ? 'CNI enforces NetworkPolicy:' : 'CNI ignores NetworkPolicy:');
  console.log('src/dst'.padEnd(10) + names.map(n => n.padEnd(10)).join(''));
  for (const src of names) {
    const row = names.map(d => (src === d ? '-' : allowed(src, d, enforced) ? 'ok' : 'BLOCK').padEnd(10));
    console.log(src.padEnd(10) + row.join(''));
  }
  console.log('');
}

# preview on a namespace that has running Pods: nothing changes
$ kubectl label --dry-run=server --overwrite ns abcl-kx \
    pod-security.kubernetes.io/enforce=restricted
Warning: existing pods in namespace "abcl-kx" violate the new
PodSecurity enforce level "restricted:latest"
Warning: frontend (and 2 other pods): allowPrivilegeEscalation != false,
unrestricted capabilities, runAsNonRoot != true, seccompProfile
namespace/abcl-kx labeled (server dry run)

# baseline enforced, restricted only warned and audited
$ kubectl label ns abcl-kx pod-security.kubernetes.io/enforce=baseline \
    pod-security.kubernetes.io/warn=restricted \
    pod-security.kubernetes.io/audit=restricted --overwrite
namespace/abcl-kx labeled

# privileged Pod, server dry run only: it never runs
$ kubectl apply --dry-run=server -f 30-privileged.yaml
Error from server (Forbidden): error when creating "30-privileged.yaml":
pods "too-powerful" is forbidden: violates PodSecurity "baseline:latest":
privileged (container "app" must not set securityContext.privileged=true)

# a plain Pod passes baseline, but restricted warns
$ kubectl apply --dry-run=server -f 31-plain.yaml
Warning: would violate PodSecurity "restricted:latest":
allowPrivilegeEscalation != false (…), unrestricted capabilities (…),
runAsNonRoot != true (…), seccompProfile (…)
pod/plain-nginx created (server dry run)

# restricted enforced, pinned to one version, a real request
$ kubectl label ns abcl-kx pod-security.kubernetes.io/enforce=restricted \
    pod-security.kubernetes.io/enforce-version=v1.34 --overwrite
namespace/abcl-kx labeled
$ kubectl apply -f 31-plain.yaml
Error from server (Forbidden): error when creating "31-plain.yaml":
pods "plain-nginx" is forbidden: violates PodSecurity "restricted:v1.34":
allowPrivilegeEscalation != false (container "nginx" must set
securityContext.allowPrivilegeEscalation=false), unrestricted capabilities
(container "nginx" must set securityContext.capabilities.drop=["ALL"]),
runAsNonRoot != true (pod or container "nginx" must set
securityContext.runAsNonRoot=true), seccompProfile (pod or container "nginx"
must set securityContext.seccompProfile.type to "RuntimeDefault" or
"Localhost")
$ kubectl get pods -n abcl-kx
No resources found in abcl-kx namespace.

apiVersion: v1
kind: Pod
metadata:
  name: hardened
  namespace: abcl-kx
spec:
  serviceAccountName: report-app
  securityContext:
    runAsNonRoot: true
    runAsUser: 1000
    runAsGroup: 1000
    seccompProfile:
      type: RuntimeDefault
  containers:
    - name: app
      image: python:3.12-alpine
      imagePullPolicy: IfNotPresent
      command: ["python", "-m", "http.server", "8080", "--directory", "/tmp"]
      ports:
        - containerPort: 8080
      securityContext:
        allowPrivilegeEscalation: false
        readOnlyRootFilesystem: true
        capabilities:
          drop: ["ALL"]
      resources:
        requests: {cpu: 20m, memory: 32Mi}
        limits: {cpu: 100m, memory: 64Mi}
      volumeMounts:
        - name: tmp
          mountPath: /tmp
  volumes:
    - name: tmp
      emptyDir:
        sizeLimit: 8Mi

$ kubectl apply -f 32-hardened.yaml
pod/hardened created
$ kubectl exec -n abcl-kx hardened -- id
uid=1000 gid=1000 groups=1000
$ kubectl exec -n abcl-kx hardened -- sh -c 'touch /etc/x; echo "touch rc=$?"
    touch /tmp/ok && echo tmp-ok
    grep -E "^(CapEff|CapBnd|NoNewPrivs|Seccomp):" /proc/self/status'
touch rc=1
touch: /etc/x: Read-only file system
tmp-ok
CapEff:	0000000000000000
CapBnd:	0000000000000000
NoNewPrivs:	1
Seccomp:	2

# restricted allows runAsNonRoot without a runAsUser, then this happens:
$ kubectl apply -f 33-rootimage.yaml      # nginx image, runs as root
pod/root-image created
$ kubectl get pod root-image -n abcl-kx
NAME         READY   STATUS                       RESTARTS   AGE
root-image   0/1     CreateContainerConfigError   0          …
$ kubectl describe pod root-image -n abcl-kx | grep Error:
… kubelet  Error: container has runAsNonRoot and image will run as root (…)

// Small part of the "restricted" Pod Security profile. Edit a spec and run again.
const PLAIN = { pod: {}, ctr: {}, volumes: ['configMap'] };
const HARDENED = {
  pod: { runAsNonRoot: true, runAsUser: 1000, seccomp: 'RuntimeDefault' },
  ctr: { allowPrivilegeEscalation: false, drop: ['ALL'], readOnlyRootFilesystem: true },
  volumes: ['emptyDir', 'projected'],
};
const RISKY = {
  pod: { hostNetwork: true }, ctr: { privileged: true, add: ['SYS_ADMIN'] },
  volumes: ['hostPath'],
};
const OK_VOLUMES = ['configMap', 'csi', 'downwardAPI', 'emptyDir', 'ephemeral',
  'persistentVolumeClaim', 'projected', 'secret'];
const CHECKS = [
  ['privileged', s => !s.ctr.privileged],
  ['hostNetwork', s => !s.pod.hostNetwork],
  ['allowPrivilegeEscalation=false', s => s.ctr.allowPrivilegeEscalation === false],
  ['capabilities.drop=ALL', s => (s.ctr.drop || []).includes('ALL')],
  ['capabilities.add', s => (s.ctr.add || []).every(c => c === 'NET_BIND_SERVICE')],
  ['runAsNonRoot=true', s => s.pod.runAsNonRoot === true],
  ['runAsUser not 0', s => s.pod.runAsUser !== 0],
  ['seccomp', s => ['RuntimeDefault', 'Localhost'].includes(s.pod.seccomp)],
  ['volume types', s => s.volumes.every(v => OK_VOLUMES.includes(v))],
];
function report(name, spec) {
  const bad = CHECKS.filter(([, ok]) => !ok(spec)).map(([label]) => label);
  console.log(name + ': ' + (bad.length ? 'REJECT, ' + bad.length + ' violation(s)' : 'PASS'));
  for (const b of bad) console.log('   - ' + b);
  if (!spec.ctr.readOnlyRootFilesystem) {
    console.log('   note: root filesystem is writable (allowed by restricted)');
  }
}
report('PLAIN', PLAIN);
report('HARDENED', HARDENED);
report('RISKY', RISKY);

apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicy
metadata:
  name: abcl-kx-no-latest-tag
spec:
  failurePolicy: Fail
  matchConstraints:
    resourceRules:
      - apiGroups: [""]
        apiVersions: ["v1"]
        operations: ["CREATE", "UPDATE"]
        resources: ["pods"]
  validations:
    - expression: "object.spec.containers.all(c, c.image.contains(':') && !c.image.endsWith(':latest'))"
      message: "image must carry an explicit tag, and that tag must not be latest"
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicyBinding
metadata:
  name: abcl-kx-no-latest-tag
spec:
  policyName: abcl-kx-no-latest-tag
  validationActions: ["Deny"]
  matchResources:
    namespaceSelector:
      matchLabels:
        kubernetes.io/metadata.name: abcl-kx

$ kubectl apply --dry-run=server -f 41-latest.yaml    # image: python:latest
The pods "lazy-tag" is invalid: : ValidatingAdmissionPolicy
'abcl-kx-no-latest-tag' with binding 'abcl-kx-no-latest-tag' denied request:
image must carry an explicit tag, and that tag must not be latest
$ kubectl apply --dry-run=server -f 42-good.yaml      # python:3.12-alpine
pod/good-tag created (server dry run)