Attention va Transformer

import os
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
from torch.utils.data import TensorDataset, DataLoader

torch.set_num_threads(4)

diagramma(
    {
        "1-qadam": (0.2, 0.2, 1.3, 0.7, "manba"),
        "...": (1.9, 0.2, 1.3, 0.7, "manba"),
        "60-qadam": (3.6, 0.2, 1.3, 0.7, "manba"),
        "RNN: 59 ta qadam\\nbosib o'tiladi": (0.2, 1.5, 4.7, 0.8, "xizmat"),
        "Attention: har qadamga\\nTO'G'RIDAN-TO'G'RI yo'l": (0.2, 2.7, 4.7, 0.8, "qayta"),
    },
    [("1-qadam", "RNN: 59 ta qadam\\nbosib o'tiladi"),
     ("60-qadam", "RNN: 59 ta qadam\\nbosib o'tiladi"),
     ("RNN: 59 ta qadam\\nbosib o'tiladi", "Attention: har qadamga\\nTO'G'RIDAN-TO'G'RI yo'l")],
    nom="yol_uzunligi.png",
    izoh="RNN da ma'lumot zanjir bo'ylab yuradi va yo'lda kuchsizlanadi. "
         "Attention da har pozitsiya orasidagi masofa — bitta qadam.",
)

torch.manual_seed(0)
T_kichik, d = 4, 8
h = torch.randn(1, T_kichik, d)

Wq, Wk, Wv = (nn.Linear(d, d, bias=False) for _ in range(3))
Q, K, V = Wq(h), Wk(h), Wv(h)
print(f"Kirish h: {tuple(h.shape)}   -> {T_kichik} pozitsiya, har biri {d} o'lchamli vektor")
print(f"Q, K, V : {tuple(Q.shape)}")
print()

ball = Q @ K.transpose(1, 2)
print("1-qadam: har Q ni har K bilan skalyar ko'paytirish (o'xshashlik balli)")
print(f"  ball shakli: {tuple(ball.shape)}  -> har pozitsiya har pozitsiyaga ball qo'yadi")
print(np.round(ball[0].detach().numpy(), 2))
print()

ball_normal = ball / d ** 0.5
print(f"2-qadam: sqrt(d) ga bo'lish (d={d}, sqrt={d**0.5:.3f})")
print("  Nega: d katta bo'lsa ballar ham katta bo'ladi, softmax esa 0 va 1 ga yopishib qoladi.")
print()

ogirlik = torch.softmax(ball_normal, dim=-1)
print("3-qadam: softmax — ballarni og'irlikka aylantirish")
print(np.round(ogirlik[0].detach().numpy(), 3))
print(f"  har qator yig'indisi: {ogirlik[0].sum(dim=-1).detach().numpy().round(4)}")
print()

chiqish = ogirlik @ V
print(f"4-qadam: og'irliklar bilan V ning o'rtachasi -> {tuple(chiqish.shape)}")
print()
print("Hammasi bitta qatorda:")
print("  softmax(Q @ K.T / sqrt(d)) @ V")

def vazifa(n, T, seed=0):
    """Faqat BIRINCHI element javobni belgilaydi. Qolgani shovqin."""
    rr = np.random.default_rng(seed)
    Xs = rr.normal(0, 1, (n, T, 1)).astype(np.float32)
    return torch.tensor(Xs), torch.tensor((Xs[:, 0, 0] > 0).astype(np.int64))

class Ketma(nn.Module):
    def __init__(self, turi, h=32):
        super().__init__()
        self.rnn = nn.RNN(1, h, batch_first=True) if turi == "rnn" else nn.LSTM(1, h, batch_first=True)
        self.fc = nn.Linear(h, 2)
    def forward(self, x):
        o, _ = self.rnn(x)
        return self.fc(o[:, -1])

class Attention(nn.Module):
    def __init__(self, T, d=32, pozitsiya=True):
        super().__init__()
        self.inp = nn.Linear(1, d)
        self.poz = nn.Parameter(torch.randn(1, T, d) * 0.02) if pozitsiya else None
        self.q, self.k, self.v = (nn.Linear(d, d) for _ in range(3))
        self.fc = nn.Linear(d, 2)
        self.ogirlik = None
    def forward(self, x):
        hh = self.inp(x)
        if self.poz is not None:
            hh = hh + self.poz
        Qx, Kx, Vx = self.q(hh), self.k(hh), self.v(hh)
        w = torch.softmax(Qx @ Kx.transpose(1, 2) / Kx.shape[-1] ** 0.5, dim=-1)
        self.ogirlik = w.detach()
        return self.fc((w @ Vx).mean(1))

T = 60
print(f"Kirishning birinchi va oxirgi qadamiga yetib boradigan gradient (T={T}):")
print()
grad_jadval = []
for nom, yasa in [("RNN", lambda: Ketma("rnn")), ("LSTM", lambda: Ketma("lstm")),
                  ("attention", lambda: Attention(T))]:
    torch.manual_seed(0)
    Xg, yg = vazifa(256, T, 1)
    Xg.requires_grad_(True)
    m = yasa()
    nn.CrossEntropyLoss()(m(Xg), yg).backward()
    g = Xg.grad.abs().mean(dim=(0, 2))
    grad_jadval.append({"model": nom, "qadam 0": f"{g[0]:.3e}", "qadam 59": f"{g[59]:.3e}",
                        "qadam59 / qadam0": f"{g[59] / g[0]:.2e}"})
show(pd.DataFrame(grad_jadval))

def oqit(yasa, T_, seed, epoxalar=60, n=2000, lr=0.005):
    torch.manual_seed(seed)
    Xtr, ytr = vazifa(n, T_, 1)
    Xte, yte = vazifa(600, T_, 2)
    m = yasa()
    opt = torch.optim.Adam(m.parameters(), lr=lr)
    krit = nn.CrossEntropyLoss()
    dl = DataLoader(TensorDataset(Xtr, ytr), batch_size=64, shuffle=True,
                    generator=torch.Generator().manual_seed(seed))
    for _ in range(epoxalar):
        m.train()
        for xb, yb in dl:
            opt.zero_grad(); krit(m(xb), yb).backward(); opt.step()
    m.eval()
    with torch.no_grad():
        return float((m(Xte).argmax(1) == yte).float().mean()), m

print("Tasodifiy javob = 0.50. Har model 3 seed bilan.")
sinov = []
for nom, yasa in [
    ("RNN", lambda: Ketma("rnn")),
    ("LSTM", lambda: Ketma("lstm")),
    ("attention (+ pozitsiya)", lambda: Attention(T)),
    ("attention (pozitsiyaSIZ)", lambda: Attention(T, pozitsiya=False)),
]:
    aniq = [oqit(yasa, T, s)[0] for s in (0, 1, 2)]
    sinov.append({"model": nom, "seedlar": str([round(a, 3) for a in aniq]),
                  "o'rtacha": round(float(np.mean(aniq)), 4),
                  "eng yomon": round(min(aniq), 4),
                  "parametrlar": sum(p.numel() for p in yasa().parameters())})
show(pd.DataFrame(sinov))

torch.manual_seed(0)
m_poz = Attention(T, pozitsiya=False)
Xs, _ = vazifa(8, T, 5)
with torch.no_grad():
    asl = m_poz(Xs)
    aralash = m_poz(Xs[:, torch.randperm(T)])     # pozitsiyalarni ARALASHTIRAMIZ

print("Pozitsiya kodlashsiz attention — kirishni aralashtirib ko'ramiz:")
print(f"  asl chiqish[0]     : {asl[0].numpy().round(5)}")
print(f"  aralash chiqish[0] : {aralash[0].numpy().round(5)}")
print(f"  maksimal farq      : {float((asl - aralash).abs().max()):.2e}")
print()
print("Chiqish o'zgarmadi. Demak model uchun ketma-ketlik — TARTIBSIZ TO'PLAM.")
print("U 'birinchi element' degan tushunchaga ega emas, shuning uchun vazifani yecha olmaydi.")

aniqlik, model = oqit(lambda: Attention(T), T, 0)
Xv, _ = vazifa(200, T, 2)
with torch.no_grad():
    model(Xv)
w = model.ogirlik.mean(dim=(0, 1)).numpy()

print(f"O'qitilgan attention modelining aniqligi: {aniqlik:.4f}")
print()
print(f"Qadam 0 ga berilgan o'rtacha e'tibor : {w[0]:.4f}")
print(f"Qolgan 59 qadamning o'rtachasi      : {w[1:].mean():.6f}")
print(f"Eng ko'p e'tibor olgan qadam        : {int(w.argmax())}")
print()
print("Model o'zi topdi: javob BIRINCHI qadamda. Biz bunga hech qanday ko'rsatma bermadik.")

rasm, (b1, b2) = plt.subplots(1, 2, figsize=(14, 4.2),
                              gridspec_kw={"width_ratios": [1.15, 1]})
b1.bar(np.arange(T), w, color="#1a73e8")
b1.set_title("Har qadamga berilgan o'rtacha e'tibor")
b1.set_xlabel("ketma-ketlik qadami"); b1.set_ylabel("og'irlik")
tasvir = b2.imshow(model.ogirlik[0].numpy(), cmap="viridis", aspect="auto")
b2.set_title("Attention matritsasi (bitta namuna)")
b2.set_xlabel("qaralayotgan qadam"); b2.set_ylabel("so'rov qadami")
plt.colorbar(tasvir, ax=b2, fraction=0.046)
plt.tight_layout()
fig("attention_ogirligi.png",
    caption="Chapda: butun e'tibor 0-qadamga to'plangan. O'ngda: attention matritsasining "
            "chap ustuni yorqin — har pozitsiya 0-qadamga qaraydi.")

print(f"{'T':>6} {'RNN vaqti (ms)':>16} {'attention (ms)':>16} {'attn matritsa hajmi':>22}")
print("-" * 64)
for Tx in [30, 60, 120, 240]:
    Xb, _ = vazifa(64, Tx, 3)
    r_m, a_m = Ketma("rnn"), Attention(Tx)
    with torch.no_grad():
        r_m(Xb); a_m(Xb)                                   # isitish
        t0 = time.perf_counter(); [r_m(Xb) for _ in range(10)]
        t_rnn = (time.perf_counter() - t0) * 100
        t0 = time.perf_counter(); [a_m(Xb) for _ in range(10)]
        t_attn = (time.perf_counter() - t0) * 100
    print(f"{Tx:>6} {t_rnn:>16.2f} {t_attn:>16.2f} {f'{Tx}x{Tx} = {Tx*Tx}':>22}")
print()
print("Matritsa hajmi ustuni: T ikki barobar oshsa, u ANIQ to'rt barobar o'sadi.")
print("Vaqt ustuni esa boshqa narsani aytadi — pastdagi izohni o'qing.")

show(pd.DataFrame([
    {"Qism": "1. Multi-head attention",
     "Vazifasi": "Bir nechta attention parallel — har biri boshqa munosabatga qaraydi"},
    {"Qism": "2. Residual ulanish (x + attn(x))",
     "Vazifasi": "Gradientga to'g'ri yo'l beradi — chuqur tarmoq uchun shart"},
    {"Qism": "3. LayerNorm",
     "Vazifasi": "Har qatlam chiqishini barqarorlashtiradi"},
    {"Qism": "4. Feed-forward (2 qatlamli MLP)",
     "Vazifasi": "Attention yig'gan ma'lumotni qayta ishlaydi"},
]))
print()
print("Shu blok N marta taxlanadi. GPT, BERT, Llama — hammasi shu naqshning variantlari.")
print("Farqlar: blok soni, o'lcham, pozitsiya kodlash turi va attention maskasi.")

class TransformerBlok(nn.Module):
    """Minimal, lekin to'liq: attention + residual + norm + feed-forward."""
    def __init__(self, d=32, boshlar=4):
        super().__init__()
        self.attn = nn.MultiheadAttention(d, boshlar, batch_first=True)
        self.norm1, self.norm2 = nn.LayerNorm(d), nn.LayerNorm(d)
        self.ff = nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(), nn.Linear(4 * d, d))
    def forward(self, x):
        a, _ = self.attn(x, x, x)
        x = self.norm1(x + a)                 # residual + norm
        return self.norm2(x + self.ff(x))     # residual + norm

blok = TransformerBlok()
namuna = torch.randn(2, 10, 32)
print(f"Kirish : {tuple(namuna.shape)}")
print(f"Chiqish: {tuple(blok(namuna).shape)}   (shakl saqlanadi — shuning uchun taxlash mumkin)")
print(f"Parametrlar: {sum(p.numel() for p in blok.parameters())}")
print()
print("nn.MultiheadAttention — biz qo'lda yozgan formulaning tayyor, tezroq versiyasi.")

show(pd.DataFrame([
    {"Vazifa": "Matn klassifikatsiya", "Yondashuv": "Tayyor model + oxirgi qatlam (fine-tuning)"},
    {"Vazifa": "Savol-javob o'z hujjatlaringiz ustida", "Yondashuv": "RAG — qidirish + LLM"},
    {"Vazifa": "O'z uslubingizda matn", "Yondashuv": "LoRA fine-tuning"},
    {"Vazifa": "Rasmdan matn (OCR)", "Yondashuv": "Vision transformer + dekoder"},
    {"Vazifa": "Vaqt qatori", "Yondashuv": "Avval 34-bob: differensiyalash va klassik usullar"},
]))
print()
print("Muhim: bu vazifalarning hech birida transformer ni NOLDAN o'qitmaysiz.")
print("Tayyor model olinadi va moslashtiriladi. Buning yo'li — 38-bobda (LoRA, QLoRA).")

show(pd.DataFrame([
    {"Tushuncha": "Attention formulasi", "Bitta jumla": "softmax(Q @ K.T / sqrt(d)) @ V"},
    {"Tushuncha": "Q, K, V", "Bitta jumla": "So'rov, kalit, qiymat — uchtasi ham chiziqli proyeksiya"},
    {"Tushuncha": "Nega sqrt(d)", "Bitta jumla": "Softmax ni to'yinishdan saqlaydi"},
    {"Tushuncha": "Yo'l uzunligi", "Bitta jumla": "Har pozitsiya orasida bitta qadam — 59 emas"},
    {"Tushuncha": "Gradient oqimi", "Bitta jumla": "O'lchandi: nisbat birga yaqin, RNN da milliardlar"},
    {"Tushuncha": "Pozitsiya kodlash", "Bitta jumla": "O'lchandi: busiz model tartibni ko'rmaydi"},
    {"Tushuncha": "Tushuntirish", "Bitta jumla": "Og'irliklar modelning qarashini ko'rsatadi"},
    {"Tushuncha": "O(T kvadrat)", "Bitta jumla": "Matritsa 4 barobar o'sadi — to'siq XOTIRADA, tezlikda emas"},
    {"Tushuncha": "Parallellik", "Bitta jumla": "O'lchandi: kichik T da attention RNN dan tezroq"},
    {"Tushuncha": "Transformer bloki", "Bitta jumla": "Attention + residual + norm + feed-forward"},
]))