ক্যাপস্টোন — বাংলা পণ্য-রিভিউ থেকে একটি পূর্ণ AI প্রকল্প

# bazar_text.py — বাজারঘরের রিভিউ পরিষ্কার আর feature বানানোর ছোট module (ক্যাপস্টোন, পর্ব ৭৩)
import re, unicodedata
import numpy as np
import pandas as pd

BN = f"{chr(0x0980)}-{chr(0x09FF)}"                         # পুরো বাংলা Unicode block
EMO = f"{chr(0x1F300)}-{chr(0x1FAFF)}{chr(0x2600)}-{chr(0x27BF)}{chr(0x2B50)}"
ZW = dict.fromkeys([0x200B, 0x200C, 0x200D, 0xFEFF])       # অদৃশ্য অক্ষর → মুছে ফেলা
BN2EN = str.maketrans("০১২৩৪৫৬৭৮৯", "0123456789")
TOK = re.compile(rf"[\w{BN}]+|[{EMO}]")                    # একটি শব্দ, নয়তো একটি emoji

def clean(t):
    """NFC, zero-width মোছা, বাংলা অঙ্ক → 0-9, বাড়তি space বাদ, lower."""
    t = unicodedata.normalize("NFC", t).translate(ZW).translate(BN2EN)
    return re.sub(r"\s+", " ", t).strip().lower()

def tok(t):
    return TOK.findall(t)

def extra_feats(s):
    """দৈর্ঘ্য (log), '!' আর emoji-র সংখ্যা, কোনো অঙ্ক আছে কি না।"""
    s = pd.Series(s)
    return np.c_[np.log1p(s.str.len()), s.str.count("!"), s.str.count(f"[{EMO}]"),
                 s.str.contains(r"\d")].astype(float)

def script_of(t):
    return "bangla" if re.search(f"[{BN}]", t) else "latin"

import re
import numpy as np
import pandas as pd
from bazar_text import clean, tok, extra_feats, script_of, BN, EMO   # পাতার ভেতরের module

raw = pd.read_csv("reviews.csv", keep_default_na=False)   # ফাঁকা ঘর "" থাকুক, NaN নয়
log, df = [], raw.copy()
def step(name, n): log.append((name, int(n)))

new = df["text"].map(clean)
step("text: NFC, zero-width, space, অঙ্ক, lower", (new != df["text"]).sum())
df["text"] = new
new = df["issue"].str.strip().str.lower()
step("issue: বড়-ছোট হাতের অক্ষর এক করা", (new != df["issue"]).sum())
df["issue"] = new
bad = df["text"].eq(""); step("খালি লেখা, বাদ", bad.sum()); df = df[~bad]
bad = df["sentiment"].eq(""); step("sentiment নেই, বাদ", bad.sum()); df = df[~bad]
df["date"] = pd.to_datetime(df["date"])

print("কাঁচা:", raw.shape, list(raw.columns))
print("cleaning log:")
for name, n in log: print(f"  {n:3d}  {name}")
print("এখন সারি:", len(df), "| এক-অক্ষরের য় (U+09DF):",
      raw["text"].str.count(chr(0x09DF)).sum(), "→", df["text"].str.count(chr(0x09DF)).sum())
print("label:", sorted(df["sentiment"].unique()), sorted(df["issue"].unique()))
print("তারিখ:", df["date"].min().date(), "থেকে", df["date"].max().date())

from sklearn.feature_extraction.text import TfidfVectorizer

df["key"] = df["text"].map(lambda t: re.sub(rf"[^\w{BN}]+", "", t))  # যতিচিহ্ন, emoji, space বাদ
same = df[df.duplicated("key", keep=False)]
odd = same.groupby("key").filter(lambda g: g["sentiment"].nunique() > 1 or g["issue"].nunique() > 1)
print("একই key-র সারি", len(same), "| বাদ যাবে", int(df.duplicated("key").sum()))
print("label মেলে না:"); print(odd[["id", "date", "text", "sentiment"]].to_string(index=False, header=False))
CUT = pd.Timestamp("2026-07-01")                       # অংশ ১-এ, ডেটা দেখার আগে ঠিক করা
span = same.groupby("key")["date"].agg(lambda s: s.min() < CUT <= s.max()).sum()
d1 = df.drop_duplicates("key", keep="first").reset_index(drop=True)   # আগেরটি থাকে

E = TfidfVectorizer(analyzer="char_wb", ngram_range=(2, 4)).fit_transform(d1["text"])
S = (E @ E.T).toarray(); np.fill_diagonal(S, 0)       # সারি L2 = 1, তাই dot = cosine
cust = d1["customer"].to_numpy(dtype=object)
a, b = np.where(np.triu(S >= 0.85) & (cust[:, None] == cust[None, :]))
for i, j in zip(a, b):
    print(f"near {S[i, j]:.2f} {d1.id[i]} {d1.date[i]:%d %b} → {d1.id[j]} {d1.date[j]:%d %b}: {d1.text[j]}")
span += int(sum((d1.date[i] < CUT) & (d1.date[j] >= CUT) for i, j in zip(a, b)))
d2 = d1.drop(index=b).reset_index(drop=True)

train = d2[d2["date"] < CUT].reset_index(drop=True)
test = d2[d2["date"] >= CUT].reset_index(drop=True)    # 🔒 অংশ ৭-এর আগে আর খোলা হবে না
print(f"সারি {len(df)} → {len(d1)} → {len(d2)} | July-র দুই পাশে ছড়ানো নকল: {span}")
print(f"train {len(train)} ({train.date.min():%d %b}–{train.date.max():%d %b})"
      f" | test {len(test)} ({test.date.min():%d %b}–{test.date.max():%d %b})")

import matplotlib.pyplot as plt

tr = train.copy()                                      # EDA শুধু train-এ
tr["script"] = tr["text"].map(script_of)
tr["n_char"] = tr["text"].str.len()
print(pd.crosstab(tr["sentiment"], tr["issue"], margins=True))
print("\nলিপি:", tr["script"].value_counts().to_dict())
print(tr.groupby("script")["n_char"].agg(["count", "mean", "min", "max"]).round(1))
neg = tr["text"].str.contains(rf"(?<![\w{BN}])(?:না|নেই|na|nai)(?![\w{BN}])")
print("\n'না/নেই/na/nai' আছে:", int(neg.sum()), "| তার positive:",
      int((neg & (tr["sentiment"] == "positive")).sum()))
print("emoji 🙄/👏 (ব্যঙ্গের সন্দেহ):", tr.loc[tr["text"].str.contains("🙄|👏"), "sentiment"].value_counts().to_dict())
month = tr.groupby(tr["date"].dt.month)["issue"].value_counts(normalize=True).unstack()

fig, ax = plt.subplots(1, 2, figsize=(9, 3.2))
for s, c in [("bangla", "#1f77b4"), ("latin", "#f59e0b")]:
    ax[0].hist(tr.loc[tr.script == s, "n_char"], bins=12, alpha=0.6, color=c, label=s)
ax[0].set(title="Review length (train)", xlabel="characters", ylabel="reviews"); ax[0].legend()
month.plot(ax=ax[1], marker="o", color=["#dc2626", "#64748b", "#16a34a"])
ax[1].set(title="Issue share by month (train)", xlabel="month (2026)", ylabel="share")
plt.show()

from sklearn.pipeline import FeatureUnion, make_pipeline
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import StratifiedKFold, cross_val_predict

cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)   # সব জায়গায় একই fold

def suspects(task, k=6):
    words_chars = FeatureUnion([
        ("w", TfidfVectorizer(tokenizer=tok, token_pattern=None)),
        ("c", TfidfVectorizer(analyzer="char_wb", ngram_range=(2, 4)))])
    P = cross_val_predict(make_pipeline(words_chars, MultinomialNB()), train["text"],
                          train[task], cv=cv, method="predict_proba")   # প্রতিটি সারি না-দেখা model-এ
    classes = np.array(sorted(train[task].unique()))
    p_given = P[np.arange(len(train)), np.searchsorted(classes, train[task])]
    top = np.argsort(p_given, kind="stable")[:k]
    return pd.DataFrame({"id": train["id"][top], "p": p_given[top].round(2),
                         "label": train[task][top], "text": train["text"][top].str[:44]})

for task in ["sentiment", "issue"]:
    print(f"\n{task}: নিজের label-এ সবচেয়ে কম আস্থা")
    print(suspects(task).to_string(index=False))

from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import FunctionTransformer, MaxAbsScaler

FIX = {"r009": ("sentiment", "positive"), "r063": ("sentiment", "positive"),   # হাতে পড়ে নিশ্চিত
       "r021": ("issue", "none"), "r148": ("issue", "quality"), "r200": ("issue", "delivery")}
for rid, (col, val) in FIX.items():
    train.loc[train["id"] == rid, col] = val

def make_features(word_ngrams=(1, 1)):
    return ColumnTransformer([
        ("word", TfidfVectorizer(tokenizer=tok, token_pattern=None, ngram_range=word_ngrams), "text"),
        ("char", TfidfVectorizer(analyzer="char_wb", ngram_range=(2, 4)), "text"),
        ("num", make_pipeline(FunctionTransformer(extra_feats), MaxAbsScaler()), "text")])

F = make_features().fit(train[["text"]])               # এখানে শুধু দেখার জন্য; CV-তে প্রতি fold-এ নতুন fit
X = F.transform(train[["text"]])
parts = {n: len(t.vocabulary_) for n, t in F.named_transformers_.items() if n != "num"}
print("ঠিক করা label:", list(FIX), "| X:", X.shape, parts, "+ num 4")
print(f"শূন্য নয় এমন ঘর: {X.nnz / (X.shape[0] * X.shape[1]):.1%}")
ex = pd.DataFrame({"text": ["Product ta ভালো না!! 😡"]})
x = F.transform(ex)
print("\nউদাহরণ:", ex.text[0], "→ token", tok(ex.text[0].lower()))
print("word ঘর", F.named_transformers_["word"].transform(ex.text).nnz, "| char ঘর",
      F.named_transformers_["char"].transform(ex.text).nnz, "| num", x[:, -4:].toarray().round(3)[0])

from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.dummy import DummyClassifier
from sklearn.model_selection import cross_validate
from sklearn.metrics import f1_score, recall_score, make_scorer

KW = {"delivery": {"ডেলিভারি", "দেরি", "কুরিয়ার", "রাইডার", "পার্সেল", "delivery", "deri", "late", "courier", "rider", "parcel"},
      "quality": {"মান", "নষ্ট", "ভাঙা", "সাইজ", "মাপ", "রং", "নকল", "পচা", "পোকা", "quality", "noshto", "size", "rong", "fake"},
      "negative": {"খারাপ", "নষ্ট", "দেরি", "না", "নেই", "ভুল", "kharap", "noshto", "deri", "late", "na", "nai", "😡", "😞"}}
def rule(t, task):                                     # শেখে না, তাই পুরো train-এ একবার মাপা
    w = set(tok(t))
    if task == "sentiment": return "negative" if w & KW["negative"] else "positive"
    return "delivery" if w & KW["delivery"] else "quality" if w & KW["quality"] else "none"

R_DEL = make_scorer(recall_score, labels=["delivery"], average="macro")
models = {"Dummy": DummyClassifier, "MultinomialNB": MultinomialNB,
          "LogisticReg": lambda: LogisticRegression(max_iter=2000), "LinearSVC": LinearSVC}
for task in ["sentiment", "issue"]:
    y = train[task]
    sc = {"F1": "f1_macro"} | ({"R_del": R_DEL} if task == "issue" else {})
    p = train["text"].map(lambda t: rule(t, task))
    extra = f"  R_del {recall_score(y, p, labels=['delivery'], average='macro'):.3f}" if task == "issue" else ""
    print(f"\n{task:9}  {'keyword rule':13}  F1 {f1_score(y, p, average='macro'):.3f}{extra}")
    for name, M in models.items():
        r = cross_validate(make_pipeline(make_features(), M()), train[["text"]], y, cv=cv, scoring=sc)
        print(f"{task:9}  {name:13}  " + "  ".join(
            f"{k} {r['test_' + k].mean():.3f} ± {r['test_' + k].std():.3f}" for k in sc))

from sklearn.model_selection import GridSearchCV

grid = {"linearsvc__C": [0.3, 1, 3], "linearsvc__class_weight": [None, "balanced"]}
best = {}
for task in ["sentiment", "issue"]:
    gs = GridSearchCV(make_pipeline(make_features(), LinearSVC()), grid, cv=cv,
                      scoring="f1_macro").fit(train[["text"]], train[task])
    res = pd.DataFrame(gs.cv_results_)
    res["setting"] = [f"C={p['linearsvc__C']}, {p['linearsvc__class_weight']}" for p in res["params"]]
    print(f"\n{task}")
    print(res[["setting", "mean_test_score", "std_test_score", "rank_test_score"]]
          .round(3).to_string(index=False, header=["setting", "F1", "sd", "rank"]))
    best[task] = {k.split("__")[1]: v for k, v in gs.best_params_.items()}
print("\nবাছাই:", best)

folds = list(cv.split(train, train["issue"]))          # একই ৫টি ভাগ, দুই সংস্করণেই
def oof(task, ngrams):                                  # প্রতিটি সারির অনুমান, না-দেখা model থেকে
    pipe = make_pipeline(make_features(ngrams), LinearSVC(**best[task]))
    return cross_val_predict(pipe, train[["text"]], train[task], cv=folds)
def tags(t):
    out = [n for n, pat in [("negation", rf"(?<![\w{BN}])(?:না|নেই|na|nai|hoyni)(?![\w{BN}])"),
                            ("sarcasm?", "🙄|👏"), ("mixed", "শুধু|তবে|কিন্তু|kintu|shudhu")] if re.search(pat, t)]
    return ",".join(out + ([] if re.search(f"[{BN}]", t) else ["latin"])) or "-"

for task in ["sentiment", "issue"]:
    y = train[task].to_numpy()
    p1, p2 = oof(task, (1, 1)), oof(task, (1, 2))
    wrong = np.flatnonzero(p1 != y)
    print(f"\n{task}: unigram-এ ভুল {len(wrong)}টি  →  tag:",
          pd.Series([t for i in wrong for t in tags(train.text[i]).split(",")]).value_counts().to_dict())
    for i in wrong[:6]:
        print(f"  {train.id[i]} {y[i][:3]}→{p1[i][:3]} [{tags(train.text[i])}] {train.text[i][:40]}"
              + (f"  (sentiment: {train.sentiment[i]})" if task == "issue" else ""))
    f = lambda p: np.array([f1_score(y[te], p[te], average="macro") for _, te in folds])
    d = f(p2) - f(p1)
    print(f"  +bigram: সারাল {int(((p1 != y) & (p2 == y)).sum())}, নতুন ভুল {int(((p1 == y) & (p2 != y)).sum())}"
          f" | fold-ভেদে F1 পার্থক্য {np.round(d, 3)} গড় {d.mean():+.3f}")
    if task == "issue":
        r = lambda p: recall_score(y, p, labels=["delivery"], average="macro")
        print(f"  delivery recall (সব OOF): {r(p1):.3f} → {r(p2):.3f}")

from sklearn.calibration import CalibratedClassifierCV
from sklearn.metrics import confusion_matrix, classification_report

train.loc[train["id"] == "r069", "sentiment"] = "positive"       # অংশ ৬-এর loop-এ পাওয়া
final = {t: make_pipeline(make_features(), CalibratedClassifierCV(LinearSVC(**best[t]), cv=5))
         .fit(train[["text"]], train[t]) for t in ["sentiment", "issue"]}

pred = {t: final[t].predict(test[["text"]]) for t in final}     # 🔒 test খোলা হলো — একবারই
rng = np.random.default_rng(42)
def boot(y, p, cls, n=2000):                            # macro F1 আর একটি শ্রেণির recall, n বার
    idx = rng.integers(0, len(y), (n, len(y))); Y, P = y[idx], p[idx]
    f1s = [2 * ((Y == c) & (P == c)).sum(1) / np.maximum((Y == c).sum(1) + (P == c).sum(1), 1)
           for c in np.unique(y)]
    rec = ((Y == cls) & (P == cls)).sum(1) / np.maximum((Y == cls).sum(1), 1)
    return np.percentile(np.mean(f1s, 0), [2.5, 97.5]), np.percentile(rec, [2.5, 97.5])

for t, cls in [("sentiment", "negative"), ("issue", "delivery")]:
    y, p = test[t].to_numpy(), pred[t]
    labs = sorted(set(y))
    (lo, hi), (rlo, rhi) = boot(y, p, cls)
    print(f"\n{t} (test n={len(y)}): macro F1 {f1_score(y, p, average='macro'):.3f}"
          f"  95% CI [{lo:.3f}, {hi:.3f}] | {cls} recall CI [{rlo:.3f}, {rhi:.3f}]")
    print(pd.DataFrame(confusion_matrix(y, p, labels=labs), index=labs, columns=labs))
    print(classification_report(y, p, digits=3).split("\n\n")[1])

te = test.assign(script=test["text"].map(script_of))
rows = []
for g, part in te.groupby("script"):
    m = te["script"].eq(g).to_numpy()
    for t in final:
        y, p = part[t].to_numpy(), pred[t][m]
        rows.append({"group": g, "n": len(part), "task": t,
                     "F1": f1_score(y, p, average="macro"),
                     "acc": (y == p).mean(), "ভুল": int((y != p).sum())})
    y, p = part["issue"].to_numpy(), pred["issue"][m]
    rows[-1]["R_del"] = recall_score(y, p, labels=["delivery"], average="macro")
    rows[-1]["n_del"] = int((y == "delivery").sum())
print(pd.DataFrame(rows).round(3).to_string(index=False))

print("\ntest-এর ভুলগুলো (শুধু পড়া, model আর বদলাবে না):")
for t in final:
    for i in np.flatnonzero(test[t].to_numpy() != pred[t]):
        print(f"  {t[:4]} {test[t][i][:3]}→{pred[t][i][:3]} [{tags(test.text[i])}] {test.text[i][:44]}")

Q = pd.read_csv("queries.csv")                         # ২০টি label দেওয়া প্রশ্ন (topic, query)
index = train[["id", "text", "topic"]].reset_index(drop=True)   # "অতীতের রিভিউ" = শুধু train
vecs = {"word": TfidfVectorizer(tokenizer=tok, token_pattern=None),
        "char": TfidfVectorizer(analyzer="char_wb", ngram_range=(2, 4))}
D = {n: v.fit_transform(index["text"]) for n, v in vecs.items()}    # প্রতিটি সারি L2 = 1

def top3(name, q):
    s = (D[name] @ vecs[name].transform([clean(q)]).T).toarray().ravel()   # cosine
    return np.argsort(-s, kind="stable")[:3], s

def recall_at_3(name):                                  # প্রাসঙ্গিক = একই topic-এর রিভিউ
    out = []
    for topic, q in zip(Q["topic"], Q["query"]):
        rel = index["topic"].to_numpy() == topic
        out.append(rel[top3(name, q)[0]].sum() / min(3, rel.sum()))
    return np.array(out)

R = pd.DataFrame({n: recall_at_3(n) for n in vecs}, index=Q["query"].str[:28])
print(R.round(2).to_string())
print("\nগড় recall@3:", R.mean().round(3).to_dict(), "| ০ পাওয়া প্রশ্ন:", (R == 0).sum().to_dict())
q = "কুরিয়ারকে ফোনে পাওয়া যায় না"
top, s = top3("char", q)
print("\n", q)
for i in top: print(f"  {s[i]:.3f} {index.topic[i]:10} {index.text[i][:44]}")

MODEL, REV = "intfloat/multilingual-e5-small", "614241f622f53c4eeff9890bdc4f31cfecc418b3"
m = SentenceTransformer(MODEL, revision=REV, device="cpu")
P = m.encode(["passage: " + t for t in index.text], normalize_embeddings=True)
def scores(name, q):
    if name == "e5": return P @ m.encode(["query: " + clean(q)], normalize_embeddings=True)[0]
    return (D[name] @ vecs[name].transform([clean(q)]).T).toarray().ravel()

import os, joblib

joblib.dump({"models": final, "search": vecs["char"], "D": D["char"], "index": index,
             "version": "triage-v1, 2026-11"}, "triage.joblib")
print("ফাইল:", round(os.path.getsize("triage.joblib") / 1024), "KB")
bundle = joblib.load("triage.joblib")                  # pickle: শুধু নিজের বানানো ফাইল খুলুন

def predict_review(text, k=3, b=bundle):
    t = clean(text)
    out = {}
    for task, m in b["models"].items():
        pr = m.predict_proba(pd.DataFrame({"text": [t]}))[0]
        out[task] = (m.classes_[pr.argmax()], round(float(pr.max()), 2))
    s = (b["D"] @ b["search"].transform([t]).T).toarray().ravel()
    out["similar"] = [(b["index"].id[i], round(float(s[i]), 2), b["index"].text[i][:34])
                      for i in np.argsort(-s, kind="stable")[:k]]
    return out

for r in ["parcel 9 din por ashlo, khub birokto 😡", "রাইডার খুব খারাপ ব্যবহার করলেন",
          "মাপ একদম ঠিক, রং ছবির মতোই 👍"]:
    o = predict_review(r)
    print(f"\n{r}\n  sentiment {o['sentiment']} | issue {o['issue']}")
    for sid, sc, txt in o["similar"]: print(f"   {sid} {sc:.2f} {txt}")

# Model card — বাজারঘর রিভিউ-triage v1 (কাল্পনিক)
## উদ্দেশ্য
- রিভিউয়ে sentiment, issue আর ৩টি কাছের রিভিউ দেখিয়ে কর্মীকে সাহায্য; সিদ্ধান্ত মানুষের
## যে কাজে নিষেধ
- গ্রাহককে ব্লক, রিভিউ মোছা, কর্মী বা কুরিয়ারের মূল্যায়ন
## Model আর ডেটা
- word + char TF-IDF + 4টি সংখ্যা; LinearSVC + sigmoid calibration (scikit-learn 1.8.0)
- train: 193টি কাল্পনিক রিভিউ, Jan–Jun 2026, 6টি label হাতে সারানো (label_fixes.csv)
## Test (Jul–Aug 2026, n = 90, একবার; বন্ধনীতে 95% bootstrap CI)
- sentiment macro F1 0.936 [0.877, 0.986]
- issue macro F1 0.884 [0.816, 0.945]; delivery recall 0.778 [0.634, 0.912]
- খোঁজ: recall@3 0.533 (char TF-IDF, 20টি প্রশ্ন)
## দলভিত্তিক ফল (test)
- বাংলা হরফ n = 55: issue F1 0.853, delivery recall 0.636 (22টির মধ্যে)
- রোমান হরফ n = 35: issue F1 0.935, delivery recall 1.000 (14টির মধ্যে)
## জানা সীমা
- বাংলা হরফের রাইডার ও কুরিয়ারের অভিযোগ মিস করে, কখনো উঁচু আস্থায়
- ব্যঙ্গ, নতুন নাম, লম্বা মিশ্র বাক্যে দুর্বল; আস্থা মানে সঠিকতা নয়; ছোট, বানানো ডেটা
## নজরদারি
- প্রতি সপ্তাহে 50টি নমুনা হাতে label; delivery recall 0.85-এর নিচে → পর্যালোচনা
- version: triage-v1 · data sha256 87a595b822f4 · uv.lock · মালিক: গ্রাহকসেবা-ডেটা দল