Deep learning coding ইন্টারভিউ — NumPy আর PyTorch দিয়ে নিজে লেখা

import numpy as np
rng = np.random.default_rng(0)
N, D, H, C = 6, 4, 5, 3                       # batch, input, hidden, class
X, y = rng.normal(size=(N, D)), rng.integers(0, C, N)
P = {"W1": rng.normal(0, .5, (D, H)), "b1": np.zeros(H),
     "W2": rng.normal(0, .5, (H, C)), "b2": np.zeros(C)}

def loss_and_grads(P):
    h = X @ P["W1"] + P["b1"]; a = np.maximum(h, 0)        # (N, H)
    z = a @ P["W2"] + P["b2"]                               # logits (N, C)
    z = z - z.max(1, keepdims=True)                         # স্থির softmax
    logp = z - np.log(np.exp(z).sum(1, keepdims=True))
    L = -logp[np.arange(N), y].mean()
    dz = np.exp(logp); dz[np.arange(N), y] -= 1; dz /= N    # softmax − one-hot
    dW2, db2 = a.T @ dz, dz.sum(0)
    dh = (dz @ P["W2"].T) * (h > 0)                         # ReLU-এর gate
    return L, {"W1": X.T @ dh, "b1": dh.sum(0), "W2": dW2, "b2": db2}

L, G = loss_and_grads(P)
print(f"loss {L:.4f} | ln(C) = {np.log(C):.4f}")
for k in P:                                  # কেন্দ্রীয় finite difference
    num = np.zeros_like(P[k])
    for i in np.ndindex(P[k].shape):
        old = P[k][i]
        P[k][i] = old + 1e-5; Lp = loss_and_grads(P)[0]
        P[k][i] = old - 1e-5; Lm = loss_and_grads(P)[0]
        P[k][i] = old; num[i] = (Lp - Lm) / 2e-5
    rel = np.abs(num - G[k]).max() / max(np.abs(num).max(), np.abs(G[k]).max())
    print(f"{k}: shape {G[k].shape}, max rel err {rel:.1e}")

import numpy as np
rng = np.random.default_rng(1)
x = rng.normal(5, 3, (8, 3))                  # batch 8, feature 3
gamma, beta, eps = np.array([1., 2., .5]), np.zeros(3), 1e-5

def bn_forward(x, gamma, beta):
    mu, var = x.mean(0), x.var(0)             # batch-এর গড় আর biased variance
    xhat = (x - mu) / np.sqrt(var + eps)
    return gamma * xhat + beta, (xhat, var)

def bn_backward(dout, cache, gamma):
    xhat, var = cache; n = len(xhat)
    dxhat = dout * gamma
    dx = (n * dxhat - dxhat.sum(0) - xhat * (dxhat * xhat).sum(0)) / (n * np.sqrt(var + eps))
    return dx, (dout * xhat).sum(0), dout.sum(0)   # dx, dgamma, dbeta

out, cache = bn_forward(x, gamma, beta)
print("out mean:", out.mean(0).round(6) + 0, "| out std:", out.std(0).round(4))
w = rng.normal(size=out.shape)                # loss = sum(w * out)
dx, dg, db = bn_backward(w, cache, gamma)
f = lambda x: (w * bn_forward(x, gamma, beta)[0]).sum()
E = lambda i: np.eye(x.size)[i].reshape(x.shape) * 1e-5
num = np.array([(f(x + E(i)) - f(x - E(i))) / 2e-5 for i in range(x.size)]).reshape(x.shape)
print("dx max abs err:", f"{np.abs(num - dx).max():.1e}")
print("dx column sums:", dx.sum(0).round(10) + 0)
run_mu, run_var, m = np.zeros(3), np.ones(3), 0.1
for xb in rng.normal(5, 3, (50, 8, 3)):       # running stat: eval mode-এ এগুলোই
    run_mu = (1 - m) * run_mu + m * xb.mean(0)
    run_var = (1 - m) * run_var + m * xb.var(0, ddof=1)
print("running mean:", run_mu.round(2), "| running var:", run_var.round(2))

import numpy as np

class AdamW:
    def __init__(self, lr=0.1, b1=0.9, b2=0.999, eps=1e-8, wd=0.0, bias_correction=True):
        self.lr, self.b1, self.b2, self.eps, self.wd, self.bc = lr, b1, b2, eps, wd, bias_correction
        self.m = self.v = None; self.t = 0
    def step(self, w, g):
        if self.m is None: self.m, self.v = np.zeros_like(w), np.zeros_like(w)
        self.t += 1
        self.m = self.b1 * self.m + (1 - self.b1) * g           # প্রথম moment
        self.v = self.b2 * self.v + (1 - self.b2) * g * g       # দ্বিতীয় moment
        mh, vh = self.m, self.v
        if self.bc:                                             # শুরুর শূন্য-ঝোঁক সংশোধন
            mh, vh = mh / (1 - self.b1 ** self.t), vh / (1 - self.b2 ** self.t)
        w = w - self.lr * self.wd * w                           # decoupled weight decay
        return w - self.lr * mh / (np.sqrt(vh) + self.eps)

A = np.diag([10.0, 0.1])                     # একদিকে খাড়া, অন্যদিকে প্রায় সমতল
grad = lambda w: A @ w                       # loss = ½ wᵀAw, সর্বনিম্ন (0, 0)-তে
w0 = np.array([1.0, 1.0])
print("first step, with bias correction:", AdamW().step(w0, grad(w0)) - w0)
print("first step, without             :", AdamW(bias_correction=False).step(w0, grad(w0)) - w0)
opt, w, ws = AdamW(lr=0.05), w0.copy(), w0.copy()
for t in range(300):
    w = opt.step(w, grad(w))
    ws = ws - 0.15 * grad(ws)                # SGD: lr 0.2-এর বেশি হলে খাড়া দিকে বিস্ফোরণ
print("after 300 steps, Adam:", np.abs(w).round(4), "| SGD lr=0.15:", np.abs(ws).round(4))

import numpy as np
from scipy.signal import correlate
rng = np.random.default_rng(2)
x = rng.normal(size=(2, 3, 7, 7))            # (N, C_in, H, W)
w = rng.normal(size=(4, 3, 3, 3))            # (C_out, C_in, kH, kW)
b = rng.normal(size=4)
pad = lambda x, p: np.pad(x, ((0, 0), (0, 0), (p, p), (p, p)))

def conv_naive(x, w, b, s=1, p=0):
    N, C, H, W = x.shape; F, _, k, _ = w.shape
    xp, Ho, Wo = pad(x, p), (H + 2*p - k) // s + 1, (W + 2*p - k) // s + 1
    y = np.zeros((N, F, Ho, Wo))
    for n in range(N):
        for f in range(F):
            for i in range(Ho):
                for j in range(Wo):
                    y[n, f, i, j] = (xp[n, :, i*s:i*s+k, j*s:j*s+k] * w[f]).sum() + b[f]
    return y

def conv_im2col(x, w, b, s=1, p=0):
    F, C, k, _ = w.shape
    win = np.lib.stride_tricks.sliding_window_view(pad(x, p), (k, k), axis=(2, 3))[:, :, ::s, ::s]
    cols = win.transpose(0, 2, 3, 1, 4, 5)                 # (N, Ho, Wo, C, k, k)
    cols = cols.reshape(*cols.shape[:3], C * k * k)        # প্রতিটি জানালা এক সারি
    return (cols @ w.reshape(F, -1).T + b).transpose(0, 3, 1, 2)   # (N, F, Ho, Wo)

y1, y2 = conv_naive(x, w, b, s=2, p=1), conv_im2col(x, w, b, s=2, p=1)
print("output:", y1.shape, "| naive == im2col:", np.allclose(y1, y2))
ref = correlate(x[0], w[1], mode="valid")[0] + b[1]    # SciPy, stride 1, padding নেই
print("scipy check:", np.allclose(conv_im2col(x, w, b)[0, 1], ref))

import numpy as np
np.set_printoptions(precision=3, suppress=True)
rng = np.random.default_rng(3)
T, d = 4, 8                                    # ৪টি token, প্রতিটি d = 8
Q, K, V = rng.normal(size=(3, T, d))

def softmax(z):
    z = z - z.max(-1, keepdims=True)
    e = np.exp(z)
    return e / e.sum(-1, keepdims=True)

def attention(Q, K, V, causal=True):
    S = Q @ K.swapaxes(-1, -2) / np.sqrt(Q.shape[-1])      # (..., T, T)
    if causal:
        future = np.triu(np.ones(S.shape[-2:], dtype=bool), k=1)   # j > i
        S = np.where(future, -np.inf, S)                   # exp(−inf) = 0
    A = softmax(S)
    return A @ V, A

out, A = attention(Q, K, V)
print("weights (row = query):\n", A)
print("row sums:", A.sum(-1))
print("row 0 output == V[0]:", np.allclose(out[0], V[0]))
K2, V2 = K.copy(), V.copy(); K2[3] += 5; V2[3] -= 5    # শেষ token বদলাই
out2, _ = attention(Q, K2, V2)
print("rows 0-2 unchanged:", np.allclose(out[:3], out2[:3]), "| row 3 changed:", not np.allclose(out[3], out2[3]))
q, k = rng.normal(size=(2, 10000, 64))         # d = 64: scale ছাড়া score অনেক বড়
s = (q * k).sum(-1)
print("score std d=64, unscaled vs /sqrt(d):", round(s.std(), 2), round((s / 8).std(), 2))

import numpy as np
rng = np.random.default_rng(4)
B, T, D, H = 2, 5, 12, 3                       # batch, token, model dim, head
hd = D // H                                    # প্রতি head-এ 4
x = rng.normal(size=(B, T, D))
Wq, Wk, Wv, Wo = rng.normal(0, D ** -0.5, (4, D, D))

def softmax(z):
    e = np.exp(z - z.max(-1, keepdims=True)); return e / e.sum(-1, keepdims=True)
def split(t):                                  # (B, T, D) -> (B, H, T, hd)
    return t.reshape(B, T, H, hd).transpose(0, 2, 1, 3)
def mha(x):
    q, k, v = split(x @ Wq), split(x @ Wk), split(x @ Wv)
    A = softmax(q @ k.transpose(0, 1, 3, 2) / np.sqrt(hd))   # (B, H, T, T)
    o = (A @ v).transpose(0, 2, 1, 3).reshape(B, T, D)       # head-গুলো আবার জোড়া
    return o @ Wo

def mha_loop(x):                               # ধীর কিন্তু স্পষ্ট: প্রতিটি head আলাদা
    heads = []
    for h in range(H):
        sl = slice(h * hd, (h + 1) * hd)
        q, k, v = x @ Wq[:, sl], x @ Wk[:, sl], x @ Wv[:, sl]
        heads.append(softmax(q @ k.transpose(0, 2, 1) / np.sqrt(hd)) @ v)
    return np.concatenate(heads, -1) @ Wo

wrong = (x @ Wq).reshape(B, H, T, hd)          # বাগ: transpose ছাড়া reshape
print("split shape:", split(x @ Wq).shape, "| vectorised == loop:", np.allclose(mha(x), mha_loop(x)))
print("reshape without transpose gives same heads:", np.allclose(wrong, split(x @ Wq)))
print("params (no bias): 4 x D x D =", 4 * D * D, "for any H")

import numpy as np
rng = np.random.default_rng(5)
d, steps = 8, 6
Wq, Wk, Wv = rng.normal(0, d ** -0.5, (3, d, d))
emb = rng.normal(size=(steps, d))              # ধরি প্রতিটি নতুন token-এর embedding জানা

def softmax(z):
    e = np.exp(z - z.max(-1, keepdims=True)); return e / e.sum(-1, keepdims=True)

def full_recompute(x):                         # প্রতি ধাপে পুরো prefix আবার
    q, k, v = x @ Wq, x @ Wk, x @ Wv
    S = q @ k.T / np.sqrt(d)
    S[np.triu(np.ones(S.shape, dtype=bool), 1)] = -np.inf
    return (softmax(S) @ v)[-1]                # শুধু শেষ token-এর output লাগে

K_cache, V_cache = np.empty((0, d)), np.empty((0, d))
kv_rows = full_rows = 0; ok = []
for t in range(steps):
    x_t = emb[t:t + 1]                         # (1, d): শুধু নতুন token
    K_cache = np.vstack([K_cache, x_t @ Wk])   # আগের K, V আবার হিসাব হয় না
    V_cache = np.vstack([V_cache, x_t @ Wv])
    out = softmax((x_t @ Wq) @ K_cache.T / np.sqrt(d)) @ V_cache   # (1, t+1) weight
    kv_rows += 1; full_rows += t + 1
    ok.append(np.allclose(out[0], full_recompute(emb[:t + 1])))
print("cached == full recompute at every step:", all(ok))
print("K/V projections, cached vs full:", kv_rows, "vs", full_rows)
L, layers, heads, hd = 4096, 24, 16, 64        # উদাহরণ: fp16-এ এক sequence-এর cache
print("KV cache MB:", 2 * layers * heads * hd * L * 2 / 2**20)

from collections import Counter
corpus = {"দাম": 6, "দামি": 4, "কম": 5, "কমদামি": 3, "ভালো": 4}   # শব্দ: কতবার
words = {tuple(w) + ("</w>",): c for w, c in corpus.items()}       # অক্ষরে ভাঙা

def pair_counts(words):
    pc = Counter()
    for sym, c in words.items():
        for a, b in zip(sym, sym[1:]):
            pc[a, b] += c
    return pc

def merge(sym, pair):                          # বাঁ থেকে ডানে, একটির ওপর আরেকটি নয়
    out, i = [], 0
    while i < len(sym):
        hit = i + 1 < len(sym) and (sym[i], sym[i + 1]) == pair
        out.append(sym[i] + sym[i + 1] if hit else sym[i]); i += 2 if hit else 1
    return tuple(out)

merges = []
for step in range(6):
    pc = pair_counts(words)
    best = max(pc, key=pc.get)                 # সমান হলে আগে পাওয়া জোড়া (dict-এর ক্রম)
    merges.append(best)
    words = {merge(s, best): c for s, c in words.items()}
    print(f"merge {step + 1}: {best[0]} + {best[1]} (count {pc[best]})")
def encode(word):                    # নতুন শব্দে merge একই ক্রমে
    sym = tuple(word) + ("</w>",)
    for p in merges: sym = merge(sym, p)
    return sym
print("কমদাম ->", encode("কমদাম"), "| ভালোদাম ->", encode("ভালোদাম"))

import numpy as np
vocab = np.array(["ভালো", "দারুণ", "ঠিকঠাক", "খারাপ", "দেরি", "ফেরত"])
logits = np.array([2.0, 1.5, 0.5, 0.0, -1.0, -2.0])

def probs(z, T=1.0):                           # temperature: logit ÷ T
    z = z / T; e = np.exp(z - z.max()); return e / e.sum()

def top_k(p, k):
    keep = np.argsort(p)[::-1][:k]
    q = np.zeros_like(p); q[keep] = p[keep]; return q / q.sum()

def top_p(p, c):                               # সবচেয়ে ছোট দল যার যোগফল ≥ c
    order = np.argsort(p)[::-1]
    n = np.searchsorted(np.cumsum(p[order]), c) + 1
    q = np.zeros_like(p); q[order[:n]] = p[order[:n]]; return q / q.sum()

np.set_printoptions(precision=3, suppress=True)
for T in (0.5, 1.0, 2.0):
    print(f"T={T}:", probs(logits, T))
p = probs(logits)
print("top-k=2  :", top_k(p, 2))
print("top-p=0.8:", top_p(p, 0.8), "| kept:", int((top_p(p, 0.8) > 0).sum()))
rng = np.random.default_rng(42)
draws = rng.choice(len(p), size=2000, p=top_p(p, 0.8))   # নমুনা টানা
print("sampled share:", np.bincount(draws, minlength=len(p)) / 2000)
print("first 5 words:", " ".join(vocab[draws[:5]]))

import itertools
import numpy as np
V = ["<s>", "পণ্য", "দ্রুত", "ভালো", "এসেছে"]   # ছোট শব্দভান্ডার
P = np.array([[0.0, 0.40, 0.35, 0.10, 0.15],   # P[a, b] = আগের শব্দ a হলে পরের b-এর সম্ভাবনা
              [0.0, 0.05, 0.10, 0.25, 0.60],
              [0.0, 0.05, 0.05, 0.05, 0.85],
              [0.0, 0.30, 0.10, 0.10, 0.50],
              [0.0, 0.40, 0.30, 0.20, 0.10]])
logP = np.log(np.where(P > 0, P, 1e-12))

def score(seq):                               # শুরু <s> থেকে log-prob-এর যোগফল
    return sum(logP[a, b] for a, b in zip((0,) + seq, seq))

def beam_search(L, width):
    beams = [((), 0.0)]
    for _ in range(L):
        cand = [(s + (b,), sc + logP[s[-1] if s else 0, b]) for s, sc in beams for b in range(1, len(V))]
        beams = sorted(cand, key=lambda x: -x[1])[:width]     # সেরা width-টি রাখি
    return beams[0]

L = 2
for width in (1, 2, 3):                       # width 1 = greedy
    seq, sc = beam_search(L, width)
    print(f"width {width}: {' '.join(V[i] for i in seq):18s} log p = {sc:.3f} (p = {np.exp(sc):.3f})")
best = max(itertools.product(range(1, len(V)), repeat=L), key=score)
print("brute force:", " ".join(V[i] for i in best), f"p = {np.exp(score(best)):.3f}", f"| {4 ** L} sequences checked")

import numpy as np
rng = np.random.default_rng(7)
n_docs, n_q, d = 2000, 50, 32
docs = rng.normal(size=(n_docs, d))                       # ধরি encoder-এর embedding
gold = rng.integers(0, n_docs, n_q)                        # প্রতিটি প্রশ্নের সঠিক ডকুমেন্ট
queries = docs[gold] + rng.normal(0, 1.1, (n_q, d))       # সঠিক ডকুমেন্টের কাছাকাছি, noise-সহ

def normalize(a):
    return a / np.linalg.norm(a, axis=1, keepdims=True)

def top_k(q, D, k):
    S = normalize(q) @ normalize(D).T                      # cosine = normalize করে dot
    idx = np.argpartition(-S, k, axis=1)[:, :k]            # O(n) বাছাই, ক্রম ছাড়া
    order = np.take_along_axis(S, idx, 1).argsort(1)[:, ::-1]
    return np.take_along_axis(idx, order, 1), S            # শুধু k-টিকে sort

idx, S = top_k(queries, docs, 10)
full = np.argsort(-S, axis=1)[:, :10]                      # যাচাই: পুরো sort
print("argpartition == full sort:", (idx == full).all())
for k in (1, 5, 10):
    print(f"recall@{k}: {(idx[:, :k] == gold[:, None]).any(1).mean():.2f}")
rank = (np.argsort(-S, axis=1) == gold[:, None]).argmax(1) + 1   # সঠিকটির অবস্থান
print("MRR:", round((1 / rank).mean(), 3), "| median rank:", int(np.median(rank)))

import numpy as np
T = 1000
beta = np.linspace(1e-4, 0.02, T)              # DDPM-এর linear schedule, t = 1..T
abar = np.cumprod(1 - beta)                    # ᾱ_t, index t-1-এ
snr = abar / (1 - abar)
print("abar at t=1, 500, 1000:", abar[[0, 499, 999]].round(5))
print("t with SNR closest to 1:", int(np.argmin(np.abs(snr - 1))) + 1)

def q_sample(x0, t, rng):                      # এক লাফে: √ᾱ·x0 + √(1−ᾱ)·ε
    eps = rng.normal(size=x0.shape)
    a = abar[t - 1].reshape(-1, *[1] * (x0.ndim - 1))   # batch-এ আলাদা t
    return np.sqrt(a) * x0 + np.sqrt(1 - a) * eps, eps

rng = np.random.default_rng(0)
x0 = np.full(20000, 1.5)                       # একই "pixel" মান, অনেক নমুনা
x = x0.copy()
for t in range(1, 301):                        # ধাপে ধাপে ৩০০ বার noise
    x = np.sqrt(1 - beta[t - 1]) * x + np.sqrt(beta[t - 1]) * rng.normal(size=x.shape)
xc, _ = q_sample(x0, np.full(len(x0), 300), rng)
print(f"t=300 step-by-step: mean {x.mean():.3f}, std {x.std():.3f}")
print(f"t=300 closed form : mean {xc.mean():.3f}, std {xc.std():.3f}")
print(f"theory            : mean {np.sqrt(abar[299]) * 1.5:.3f}, std {np.sqrt(1 - abar[299]):.3f}")
imgs = rng.normal(size=(4, 1, 8, 8))           # training batch: প্রতিটি ছবিতে আলাদা t
t = rng.integers(1, T + 1, 4)
xt, eps = q_sample(imgs, t, rng)
print("batch t:", t, "| x_t shape:", xt.shape, "| target eps shape:", eps.shape)

for step in range(1, 61):
    loss = F.mse_loss(model(X).squeeze(1), y)
    if not forget: opt.zero_grad()
    loss.backward(); opt.step()

w = torch.ones(2, requires_grad=True)
for i in range(3):
    (w * 3).sum().backward(); print("after backward", i + 1, "w.grad =", w.grad.tolist())

model = nn.Sequential(nn.Linear(4, 8), nn.BatchNorm1d(8), nn.ReLU(), nn.Dropout(0.5), nn.Linear(8, 1))
x = torch.randn(16, 4)
model.train(); a, b = model(x), model(x)
print("train mode, same input twice equal:", torch.allclose(a, b))
model.eval(); a, b = model(x), model(x)
print("eval mode,  same input twice equal:", torch.allclose(a, b))
model.train(); err(lambda: model(torch.randn(1, 4)))
model.eval(); print("eval mode, batch of 1 ->", model(torch.randn(1, 4)).shape)

out = model(X)
if bug: out = torch.softmax(out, 1)        # বাগ: CrossEntropyLoss নিজেই log-softmax করে
loss = F.cross_entropy(out, y)

model = nn.Linear(3, 2)
x = torch.from_numpy(np.random.default_rng(0).normal(size=(4, 3)))
print("x dtype:", x.dtype, "| weight dtype:", model.weight.dtype)
err(lambda: model(x))
print("fixed:", model(x.float()).dtype)
logits = torch.randn(4, 3)
err(lambda: F.cross_entropy(logits, torch.tensor([0., 2., 1., 0.])))
err(lambda: F.cross_entropy(logits, torch.tensor([0, 2, 1, 0], dtype=torch.int32)))
print("int64 targets:", round(F.cross_entropy(logits, torch.tensor([0, 2, 1, 0])).item(), 4))

x = torch.arange(24).reshape(2, 3, 4)           # (B, T, C)
want = x.permute(0, 2, 1)                       # (B, C, T)
wrong = x.view(2, 4, 3)                         # একই shape, ভুল সাজানো
print("view shape:", tuple(wrong.shape), "| equal to permute:", torch.equal(wrong, want))
print("x[0]:", x[0].tolist()); print("permute[0]:", want[0].tolist()); print("view[0]:", wrong[0].tolist())
print("permuted is_contiguous:", want.is_contiguous())
err(lambda: want.view(2, -1))

Xu = torch.randn(300, 16, generator=g); yu = torch.randint(0, 2, (300,), generator=g)   # label এলোমেলো: শেখার কিছু নেই
X, y = Xu.repeat(3, 1), yu.repeat(3); grp = torch.arange(300).repeat(3)                # প্রতিটি রিভিউ ৩ বার
a, b = random_split(range(900), [720, 180], generator=torch.Generator().manual_seed(1))
run(torch.tensor(a.indices), torch.tensor(b.indices), "random row split")
perm = torch.randperm(300, generator=torch.Generator().manual_seed(1)); test_g = perm[:60]
te = torch.isin(grp, test_g)
run(torch.where(~te)[0], torch.where(te)[0], "group split      ")