Yangi dataset: 12 qadam

import pandas as pd
show(pd.DataFrame([
    {"Qadam": 1,  "Nima qilaman": "Faylni ochaman", "Chiqadigan qaror": "Nechta qator, nechta ustun — yozib qo'yaman"},
    {"Qadam": 2,  "Nima qilaman": "Target ni nomlayman", "Chiqadigan qaror": "Bitta jumla: nimani nimadan bashorat qilaman"},
    {"Qadam": 3,  "Nima qilaman": "Target turini ko'raman", "Chiqadigan qaror": "Regression / classification / clustering"},
    {"Qadam": 4,  "Nima qilaman": "Target ni SANAYMAN", "Chiqadigan qaror": "Nomutanosibmi? Bazam qanday?"},
    {"Qadam": 5,  "Nima qilaman": "HOZIR split qilaman", "Chiqadigan qaror": "Test to'plami qulflanadi"},
    {"Qadam": 6,  "Nima qilaman": "Dummy model", "Chiqadigan qaror": "O'tish chegarasi — bitta raqam"},
    {"Qadam": 7,  "Nima qilaman": "Zaharli ustunlarni topaman", "Chiqadigan qaror": "Tashlanadigan ustunlar ro'yxati"},
    {"Qadam": 8,  "Nima qilaman": "Missing va turlar", "Chiqadigan qaror": "Har ustun uchun: to'ldirish yoki tashlash"},
    {"Qadam": 9,  "Nima qilaman": "Pipeline da bitta oddiy model", "Chiqadigan qaror": "Dummy dan yaxshimi?"},
    {"Qadam": 10, "Nima qilaman": "Metrikani tanlayman", "Chiqadigan qaror": "Qaysi raqamga qarab hukm chiqaraman"},
    {"Qadam": 11, "Nima qilaman": "Xato qatorlarni O'QIYMAN", "Chiqadigan qaror": "Model nimani tushunmayapti"},
    {"Qadam": 12, "Nima qilaman": "Qaror qabul qilaman", "Chiqadigan qaror": "Bitta o'zgarish — va 9-qadamga qaytaman"},
]))

import pandas as pd

df = pd.read_csv(DATA / "uy_ijara.csv")

print("shakli:", df.shape)
print()
print(df.dtypes.to_string())

show(df.head(6))

print("ijaraga_berildi_30k:")
print("  turi             :", df.ijaraga_berildi_30k.dtype)
print("  unikal qiymatlar :", sorted(df.ijaraga_berildi_30k.unique().tolist()))
print()
print("narx_usd:")
print("  turi             :", df.narx_usd.dtype)
print("  unikal qiymatlar :", df.narx_usd.nunique(), "ta")

show(pd.DataFrame([
    {"Target qanday": "Ikki qiymat (0/1, ha/yo'q)", "Vazifa": "Binary classification"},
    {"Target qanday": "Bir nechta yorliq (3-10 xil)", "Vazifa": "Multiclass classification"},
    {"Target qanday": "Uzluksiz son (narx, harorat, vaqt)", "Vazifa": "Regression"},
    {"Target qanday": "Target YO'Q", "Vazifa": "Clustering yoki PCA"},
]))

nisbat = df.ijaraga_berildi_30k.value_counts(normalize=True).sort_index()
print("0 (berilmadi):", f"{nisbat[0]:.1%}")
print("1 (berildi)  :", f"{nisbat[1]:.1%}")
print()
print("Agar model HAMMAGA '1' desa, accuracy =", f"{nisbat[1]:.1%}")

from sklearn.model_selection import train_test_split

X = df.drop(columns=["ijaraga_berildi_30k", "narx_usd"])
y = df["ijaraga_berildi_30k"]

X_tr, X_te, y_tr, y_te = train_test_split(
    X, y, test_size=0.3, random_state=42, stratify=y
)

print(f"train: {len(X_tr)} qator, berildi {y_tr.mean():.1%}")
print(f"test : {len(X_te)} qator, berildi {y_te.mean():.1%}")

from sklearn.dummy import DummyClassifier
from sklearn.metrics import accuracy_score

dummy = DummyClassifier(strategy="most_frequent").fit(X_tr[["xonalar"]], y_tr)
baza = accuracy_score(y_te, dummy.predict(X_te[["xonalar"]]))

print(f"BAZA (hech narsa o'rganmagan model): {baza:.4f}")
print()
print("Shu raqamni yozib qo'ying. Modelingiz shundan YUQORI bo'lishi SHART.")

print("Har ustunda nechta unikal qiymat (1200 qatordan):")
for ustun in X_tr.columns:
    n = X_tr[ustun].nunique()
    belgi = "   <-- SHUBHALI: deyarli har qatorda boshqa" if n > len(X_tr) * 0.9 else ""
    print(f"  {ustun:22s} {n:5d}{belgi}")

print("narx_matn ning birinchi 3 qiymati:")
for v in X_tr.narx_matn.head(3):
    print("  ", repr(v))
print()
print("turi:", X_tr.narx_matn.dtype, "<-- object = matn")

tekshir = X_tr.assign(berildi=y_tr).groupby("berildi").agent_komissiyasi.agg(["mean", "min", "max"])
show(tekshir.reset_index())

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score

halol = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel"]

m_halol = LogisticRegression(max_iter=2000).fit(X_tr[halol], y_tr)
m_zahar = LogisticRegression(max_iter=2000).fit(X_tr[halol + ["agent_komissiyasi"]], y_tr)

a_halol = accuracy_score(y_te, m_halol.predict(X_te[halol]))
a_zahar = accuracy_score(y_te, m_zahar.predict(X_te[halol + ["agent_komissiyasi"]]))

print(f"Halol xususiyatlar      : {a_halol:.4f}")
print(f"+ agent_komissiyasi     : {a_zahar:.4f}   <-- MUKAMMAL")

tashlanadi = {
    "elon_id": "identifikator — har qatorda boshqa",
    "narx_matn": "ikkinchi target ning matn ko'rinishi",
    "agent_komissiyasi": "LEAKAGE — ijaradan keyin ma'lum bo'ladi",
}
show(pd.DataFrame([{"Ustun": k, "Nega tashlanadi": v} for k, v in tashlanadi.items()]))

yoq = X_tr.isna().sum()
yoq = yoq[yoq > 0]
if len(yoq):
    print("Bo'sh qiymatlar bor:")
    for ustun, n in yoq.items():
        print(f"  {ustun:16s} {n:3d} ta ({n/len(X_tr):.1%})")
else:
    print("Bo'sh qiymat yo'q")
print()
print("Matn ustunlar (encoding kerak):")
for u in X_tr.select_dtypes("object").columns:
    print(f"  {u:16s} {X_tr[u].nunique()} xil qiymat")

from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer

son = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel", "qurilgan_yil"]
matn = ["tuman"]

tayyorla = ColumnTransformer([
    ("son", Pipeline([
        ("toldir", SimpleImputer(strategy="median")),
        ("shkala", StandardScaler()),
    ]), son),
    ("matn", OneHotEncoder(handle_unknown="ignore"), matn),
])

quvur = Pipeline([
    ("tayyorla", tayyorla),
    ("model", LogisticRegression(max_iter=2000)),
])

quvur.fit(X_tr, y_tr)
aniqlik = accuracy_score(y_te, quvur.predict(X_te))

print(f"BAZA          : {baza:.4f}")
print(f"Birinchi model: {aniqlik:.4f}")
print(f"Farq          : {aniqlik - baza:+.4f}")

from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score

bashorat = quvur.predict(X_te)
tn, fp, fn, tp = confusion_matrix(y_te, bashorat).ravel()

print(f"TN={tn}  FP={fp}  FN={fn}  TP={tp}")
print()
print(f"Accuracy : {aniqlik:.4f}")
print(f"Precision: {precision_score(y_te, bashorat):.4f}  = TP/(TP+FP) = {tp}/({tp}+{fp})")
print(f"Recall   : {recall_score(y_te, bashorat):.4f}  = TP/(TP+FN) = {tp}/({tp}+{fn})")
print(f"F1       : {f1_score(y_te, bashorat):.4f}")

ehtimol = quvur.predict_proba(X_te)[:, 1]
tahlil = X_te.assign(haqiqat=y_te, bashorat=bashorat, ishonch=ehtimol.round(3))

xato = tahlil[tahlil.haqiqat != tahlil.bashorat]
print(f"Xato qilgan qatorlar: {len(xato)} ta ({len(xato)/len(X_te):.1%})")
print()
print("Model ENG ISHONCH bilan xato qilgan 5 qator:")
eng = xato.assign(xatolik=(xato.ishonch - xato.haqiqat).abs()).nlargest(5, "xatolik")
show(eng[["tuman", "xonalar", "maydon_m2", "metro_km", "mebel", "haqiqat", "bashorat", "ishonch"]])

jami = tahlil.groupby("tuman").size()
xatolar = xato.groupby("tuman").size()
natija = pd.DataFrame({"jami": jami, "xato": xatolar}).fillna(0)
natija["xato_ulushi"] = (natija.xato / natija.jami).round(3)
show(natija.sort_values("xato_ulushi", ascending=False).reset_index())

show(pd.DataFrame([
    {"Yo'l": "Yangi xususiyat qo'shish", "Qachon": "Xatolar bitta guruhda to'plangan",
     "Misol": "narx/maydon nisbati, metroga yaqinlik darajasi"},
    {"Yo'l": "Boshqa model", "Qachon": "Bog'lanish chiziqli emasga o'xshaydi",
     "Misol": "Decision tree, Random Forest (17-18-bob)"},
    {"Yo'l": "Ko'proq ma'lumot", "Qachon": "Train va test natijasi ikkisi ham past",
     "Misol": "Yana e'lonlar yig'ish (36-bob)"},
    {"Yo'l": "Hyperparameter tuning", "Qachon": "Model yaxshi, lekin sozlanmagan",
     "Misol": "GridSearchCV (23-bob)"},
    {"Yo'l": "Tugatish", "Qachon": "Baza dan yetarlicha yuqori, biznesga yetarli",
     "Misol": "Saqlash va xizmatga chiqarish (39-40-bob)"},
]))

def xususiyat_qosh(d):
    d = d.copy()
    d["maydon_har_xona"] = d.maydon_m2 / d.xonalar
    d["yuqori_qavat"] = (d.qavat > d.qavatlar_soni * 0.7).astype(int)
    return d

son2 = son + ["maydon_har_xona", "yuqori_qavat"]
tayyorla2 = ColumnTransformer([
    ("son", Pipeline([("toldir", SimpleImputer(strategy="median")),
                      ("shkala", StandardScaler())]), son2),
    ("matn", OneHotEncoder(handle_unknown="ignore"), matn),
])
quvur2 = Pipeline([("tayyorla", tayyorla2), ("model", LogisticRegression(max_iter=2000))])
quvur2.fit(xususiyat_qosh(X_tr), y_tr)

yangi = accuracy_score(y_te, quvur2.predict(xususiyat_qosh(X_te)))
print(f"BAZA              : {baza:.4f}")
print(f"1-urinish         : {aniqlik:.4f}")
print(f"2-urinish (+2 xus.): {yangi:.4f}")
print(f"O'zgarish         : {yangi - aniqlik:+.4f}")

diagramma(
    {
        "1. Fayl\nshape, dtypes, head":        (0, 0, 3, 1.1, "manba"),
        "2. Target ni nomlash\nbitta jumla":    (3.4, 0, 3, 1.1, "manba"),
        "3-4. Tur va sanoq\nvazifa + nomutanosiblik": (6.8, 0, 3.4, 1.1, "manba"),
        "5. SPLIT\ntest qulflanadi":            (0, 1.6, 3, 1.1, "saqlash"),
        "6. Dummy\nchegara raqami":             (3.4, 1.6, 3, 1.1, "saqlash"),
        "7-8. Zahar va missing\nustunlar qarori": (6.8, 1.6, 3.4, 1.1, "saqlash"),
        "9. Pipeline + model":                  (0, 3.2, 3, 1.1, "qayta"),
        "10. Metrika":                          (3.4, 3.2, 3, 1.1, "qayta"),
        "11. Xatolarni o'qish":                 (6.8, 3.2, 3.4, 1.1, "qayta"),
        "12. BITTA o'zgarish":                  (3.4, 4.8, 3, 1.1, "xizmat"),
    },
    [
        ("1. Fayl\nshape, dtypes, head", "2. Target ni nomlash\nbitta jumla"),
        ("2. Target ni nomlash\nbitta jumla", "3-4. Tur va sanoq\nvazifa + nomutanosiblik"),
        ("3-4. Tur va sanoq\nvazifa + nomutanosiblik", "5. SPLIT\ntest qulflanadi"),
        ("5. SPLIT\ntest qulflanadi", "6. Dummy\nchegara raqami"),
        ("6. Dummy\nchegara raqami", "7-8. Zahar va missing\nustunlar qarori"),
        ("7-8. Zahar va missing\nustunlar qarori", "9. Pipeline + model"),
        ("9. Pipeline + model", "10. Metrika"),
        ("10. Metrika", "11. Xatolarni o'qish"),
        ("11. Xatolarni o'qish", "12. BITTA o'zgarish"),
        ("12. BITTA o'zgarish", "9. Pipeline + model"),
    ],
    nom="protokol.png",
    izoh="12 qadam. Oxirgi strelka 9-qadamga qaytadi — bu aylanma, to'g'ri chiziq emas.",
)

show(pd.DataFrame([
    {"Qadam": "1-4", "Bitta jumla bilan": "Ma'lumotni tushunish — modelga tegmasdan"},
    {"Qadam": "5", "Bitta jumla bilan": "Test to'plamini qulflash (leakage shu yerda to'xtaydi)"},
    {"Qadam": "6", "Bitta jumla bilan": "Chegara raqamini olish — busiz hukm chiqarib bo'lmaydi"},
    {"Qadam": "7-8", "Bitta jumla bilan": "Zaharli ustunlarni tashlash, missing ni hal qilish"},
    {"Qadam": "9", "Bitta jumla bilan": "Pipeline da BITTA oddiy model"},
    {"Qadam": "10", "Bitta jumla bilan": "Vazifaga mos metrikani tanlash"},
    {"Qadam": "11", "Bitta jumla bilan": "Xato qatorlarni o'qish — keyingi qadam shundan chiqadi"},
    {"Qadam": "12", "Bitta jumla bilan": "BITTA o'zgarish va 9-qadamga qaytish"},
]))

show(pd.DataFrame([
    {"Model": "Dummy (baza)", "Accuracy": f"{baza:.4f}", "Izoh": "Hech narsa o'rganmagan"},
    {"Model": "Logistic + Pipeline", "Accuracy": f"{aniqlik:.4f}", "Izoh": "Bazadan +0.27"},
    {"Model": "+ 2 yangi xususiyat", "Accuracy": f"{yangi:.4f}", "Izoh": "Deyarli o'zgarish yo'q — halol natija"},
    {"Model": "+ agent_komissiyasi", "Accuracy": f"{a_zahar:.4f}", "Izoh": "LEAKAGE — yolg'on natija"},
]))

karta = """
+--------------------------------------------------------------+
|  YANGI DATASET — 12 QADAM                                    |
+--------------------------------------------------------------+
|   1  shape, dtypes, head()                                   |
|   2  "Men ___ ni ___ dan bashorat qilaman"                   |
|   3  target turi -> regression / classification              |
|   4  value_counts(normalize=True)  <- BAZANGIZ               |
|   5  train_test_split(..., stratify=y)   HOZIR!              |
|   6  DummyClassifier -> chegara raqamini yozib qo'ying       |
|   7  nunique() -> ID lar;  "bu menda bo'ladimi?" -> leakage  |
|   8  isna().sum();  object ustunlar -> OneHot                |
|   9  ColumnTransformer -> Pipeline -> fit                    |
|  10  confusion_matrix, precision, recall, F1                 |
|  11  xato qatorlarni O'QING (nlargest ishonch bo'yicha)      |
|  12  BITTA o'zgarish -> 9-qadamga qayt                       |
+--------------------------------------------------------------+
|  Mukammal natija (>0.99) = leakage signali, bayram emas      |
|  Bazani bilmasang, accuracy ma'nosiz                         |
|  Test to'plami seyfda — bir marta ochiladi                   |
+--------------------------------------------------------------+
"""
print(karta)