Ma'lumotni tayyorlash: encoding, missing, scaling

model.fit(df[["tuman", "maydon_m2"]], y)

ValueError: could not convert string to float: 'Yunusobod'

import pandas as pd
import numpy as np

df = pd.read_csv(DATA / "uy_ijara.csv")
X = df.drop(columns=["ijaraga_berildi_30k", "narx_usd"])

print("Ustun turlari:")
print(X.dtypes.to_string())
print()
print("Muammolar ro'yxati:")
matn = X.select_dtypes("object").columns.tolist()
print(f"  1. Matn ustunlar (encoding kerak): {matn}")
yoq = X.isna().sum()
print(f"  2. Bo'sh qiymatlar: {yoq[yoq > 0].to_dict()}")
print(f"  3. Takrorlangan qatorlar: {int(X.duplicated().sum())}")
shkala = X.select_dtypes(include=[np.number]).agg(["min", "max"]).T
shkala["oraliq"] = shkala["max"] - shkala["min"]
print(f"  4. Har xil shkala — eng kichik oraliq {shkala.oraliq.min():.1f}, eng katta {shkala.oraliq.max():.1f}")

show(pd.DataFrame([
    {"Turi": "Ordinal (tartibli)", "Misol": "kichik < o'rta < katta;  1-kurs < 2-kurs",
     "Tartib bormi": "HA", "Usul": "OrdinalEncoder yoki .map()"},
    {"Turi": "Nominal (tartibsiz)", "Misol": "tuman, rang, mamlakat",
     "Tartib bormi": "YO'Q", "Usul": "OneHotEncoder"},
]))

from sklearn.preprocessing import OrdinalEncoder, OneHotEncoder

tumanlar = X[["tuman"]].head(6)
print("Asl qiymatlar:")
print(tumanlar.tuman.tolist())
print()

# NOTO'G'RI: nominal ustunga raqam berish
ord_kod = OrdinalEncoder().fit_transform(X[["tuman"]])
print("OrdinalEncoder bersak:")
for t, k in zip(tumanlar.tuman, ord_kod[:6].ravel()):
    print(f"  {t:16s} -> {k:.0f}")
print()
print("MUAMMO: model endi 'Yunusobod (7) > Bektemir (0)' deb o'ylaydi.")
print("        Bu MA'NOSIZ — tumanlarni tartiblash mumkin emas.")

# TO'G'RI: nominal ustun uchun OneHot
oh = OneHotEncoder(sparse_output=False, handle_unknown="ignore")
kodlangan = oh.fit_transform(X[["tuman"]])
nomlar = oh.get_feature_names_out(["tuman"])
print(f"OneHotEncoder: 1 ustun -> {kodlangan.shape[1]} ustun")
print()
natija = pd.DataFrame(kodlangan[:6].astype(int), columns=[n.replace("tuman_", "") for n in nomlar])
natija.insert(0, "asl", tumanlar.tuman.values)
show(natija)

print("Isbot — modelga YANGI tuman kelsa:")
yangi = pd.DataFrame({"tuman": ["Uchtepa"]})
print("  handle_unknown='ignore' bilan:", oh.transform(yangi).astype(int).tolist())
qattiq = OneHotEncoder(sparse_output=False).fit(X[["tuman"]])
try:
    qattiq.transform(yangi)
except Exception as e:
    print("  handle_unknown yo'q bo'lsa  :", type(e).__name__, "->", str(e)[:58])

# Sun'iy misol: kvartira holatini toifaga ajratamiz
X_misol = X.copy()
X_misol["holat"] = pd.cut(X_misol.qurilgan_yil.fillna(X_misol.qurilgan_yil.median()),
                          bins=[0, 1990, 2010, 2100],
                          labels=["eski", "o'rta", "yangi"])
tartib = {"eski": 0, "o'rta": 1, "yangi": 2}
X_misol["holat_kod"] = X_misol.holat.map(tartib)
show(X_misol[["qurilgan_yil", "holat", "holat_kod"]].head(6))
print()
print("Bu yerda tartib MA'NOLI: yangi > o'rta > eski. Shuning uchun .map() to'g'ri.")

print(f"qurilgan_yil — bo'sh: {int(X.qurilgan_yil.isna().sum())} ta ({X.qurilgan_yil.isna().mean():.2%})")
print()
show(pd.DataFrame([
    {"Usul": "Qatorni tashlash", "Qachon": "< 1% bo'sh va ma'lumot ko'p",
     "Narxi": "Ma'lumot yo'qotadi; bo'shlik tasodifiy bo'lmasa — xatolik kiritadi"},
    {"Usul": "Mediana bilan to'ldirish", "Qachon": "Son ustun, 1-30% bo'sh",
     "Narxi": "Tarqalishni kamaytiradi; chetki qiymatlarga chidamli"},
    {"Usul": "O'rtacha bilan to'ldirish", "Qachon": "Son ustun, taqsimot simmetrik",
     "Narxi": "Chetki qiymatlar o'rtachani tortadi"},
    {"Usul": "Eng ko'p uchraydigani", "Qachon": "Matn/kategoriya ustun",
     "Narxi": "Katta kategoriyani yanada kattalashtiradi"},
    {"Usul": "'YO'Q' alohida toifa", "Qachon": "Bo'shligining O'ZI ma'noga ega",
     "Narxi": "Yo'q — ko'pincha eng yaxshi tanlov"},
    {"Usul": "Model bilan to'ldirish (KNN)", "Qachon": "Ko'p bo'sh va ustunlar bog'liq",
     "Narxi": "Sekin; leakage xavfi yuqori"},
]))

from sklearn.impute import SimpleImputer

for strat in ["median", "mean", "most_frequent"]:
    t = SimpleImputer(strategy=strat).fit_transform(X[["qurilgan_yil"]])
    print(f"  {strat:14s} -> to'ldirilgan qiymat: {t[X.qurilgan_yil.isna().values][0][0]:.1f}")
print()
print("Qo'shimcha ustun: 'bo'sh edi' belgisini saqlash")
belgili = SimpleImputer(strategy="median", add_indicator=True).fit_transform(X[["qurilgan_yil"]])
print(f"  add_indicator=True -> {belgili.shape[1]} ustun (qiymat + bo'shlik belgisi)")

shkala = X.select_dtypes(include=[np.number]).agg(["min", "max", "mean", "std"]).T.round(2)
show(shkala.reset_index().rename(columns={"index": "ustun"}))

from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler

son = ["xonalar", "maydon_m2", "metro_km", "mebel"]
for nom, sc in [("StandardScaler", StandardScaler()),
                ("MinMaxScaler", MinMaxScaler()),
                ("RobustScaler", RobustScaler())]:
    t = sc.fit_transform(X[son])
    print(f"{nom:16s} o'rtacha={t.mean():+.3f}  std={t.std():.3f}  "
          f"min={t.min():+.2f}  max={t.max():+.2f}")

show(pd.DataFrame([
    {"Scaler": "StandardScaler", "Nima qiladi": "O'rtacha 0, std 1",
     "Qachon": "Sukut bo'yicha tanlov; taqsimot normalga yaqin bo'lsa"},
    {"Scaler": "MinMaxScaler", "Nima qiladi": "0 va 1 orasiga siqadi",
     "Qachon": "Neyron tarmoq, chegara aniq kerak bo'lganda"},
    {"Scaler": "RobustScaler", "Nima qiladi": "Mediana va IQR bo'yicha",
     "Qachon": "Chetki qiymatlar KO'P bo'lsa"},
]))

show(pd.DataFrame([
    {"Model": "KNN", "Scaling": "SHART", "Nega": "Masofa hisoblaydi"},
    {"Model": "SVM", "Scaling": "SHART", "Nega": "Masofa va margin"},
    {"Model": "Logistic / Linear Regression", "Scaling": "Tavsiya etiladi", "Nega": "Gradient tezroq yaqinlashadi; koeffitsiyentlarni solishtirish mumkin bo'ladi"},
    {"Model": "Ridge / Lasso", "Scaling": "SHART", "Nega": "Jarima koeffitsiyent kattaligiga qarab beriladi"},
    {"Model": "Neyron tarmoq", "Scaling": "SHART", "Nega": "Gradient portlashi/yo'qolishi"},
    {"Model": "Decision Tree", "Scaling": "KERAK EMAS", "Nega": "Faqat 'kattami/kichikmi' deb so'raydi"},
    {"Model": "Random Forest", "Scaling": "KERAK EMAS", "Nega": "Daraxtlardan tuzilgan"},
    {"Model": "XGBoost / LightGBM", "Scaling": "KERAK EMAS", "Nega": "Daraxtlardan tuzilgan"},
]))

from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier

y = df["ijaraga_berildi_30k"]
Xs = X[son]

print(f"{'Model':<16} {'scaling YO`Q':>14} {'scaling BOR':>13} {'farq':>8}")
print("-" * 54)
for nom, m in [("KNN", KNeighborsClassifier()), ("DecisionTree", DecisionTreeClassifier(max_depth=6, random_state=42))]:
    yoq_s = cross_val_score(m, Xs, y, cv=5).mean()
    bor_s = cross_val_score(Pipeline([("sc", StandardScaler()), ("m", m)]), Xs, y, cv=5).mean()
    print(f"{nom:<16} {yoq_s:>14.4f} {bor_s:>13.4f} {bor_s-yoq_s:>+8.4f}")

print(f"Bizning faylda dublikat: {int(X.duplicated().sum())} ta")
print()
print("Lekin diqqat — DEYARLI bir xil qatorlar ham muammo:")
soxta = pd.DataFrame({
    "tuman": ["Chilonzor", "chilonzor", "Chilonzor "],
    "maydon_m2": [55.0, 55.0, 55.0],
})
print(soxta.to_string(index=False))
print()
print("duplicated() topadigan dublikat:", int(soxta.duplicated().sum()))
soxta_t = soxta.assign(tuman=soxta.tuman.str.strip().str.lower())
print("Normallashtirilgandan keyin  :", int(soxta_t.duplicated().sum()))

from sklearn.compose import ColumnTransformer
from sklearn.linear_model import LogisticRegression

son_ustun = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel", "qurilgan_yil"]
matn_ustun = ["tuman"]

tayyorla = ColumnTransformer([
    ("son", Pipeline([
        ("toldir", SimpleImputer(strategy="median")),
        ("shkala", StandardScaler()),
    ]), son_ustun),
    ("matn", Pipeline([
        ("toldir", SimpleImputer(strategy="most_frequent")),
        ("kod", OneHotEncoder(handle_unknown="ignore")),
    ]), matn_ustun),
], remainder="drop")

quvur = Pipeline([("tayyorla", tayyorla), ("model", LogisticRegression(max_iter=2000))])
ball = cross_val_score(quvur, X, y, cv=5, scoring="f1")
print(f"To'liq quvur, 5-fold CV F1: {ball.mean():.4f} ±{ball.std():.4f}")
print()
tayyorla.fit(X, y)
print(f"Kirish : {X.shape[1]} ustun")
print(f"Chiqish: {tayyorla.transform(X).shape[1]} ustun  (OneHot tumanni yoydi)")

import hashlib

def dataset_pasport(ramka, nom, target):
    xesh = hashlib.md5(pd.util.hash_pandas_object(ramka, index=True).values).hexdigest()[:16]
    satrlar = [
        f"Dataset     : {nom}",
        f"Qatorlar    : {len(ramka):,}".replace(",", " "),
        f"Ustunlar    : {ramka.shape[1]}",
        f"Target      : {target}",
        f"Xesh (md5)  : {xesh}",
        f"Bo'sh qiymat: {int(ramka.isna().sum().sum())}",
        f"Dublikat    : {int(ramka.duplicated().sum())}",
    ]
    if target in ramka:
        n = ramka[target].value_counts(normalize=True).sort_index()
        satrlar.append("Target nisbati: " + ", ".join(f"{k}={v:.1%}" for k, v in n.items()))
    return "\n".join(satrlar)

print(dataset_pasport(df, "uy_ijara.csv", "ijaraga_berildi_30k"))

show(pd.DataFrame([
    {"Muammo": "Matn ustun", "Yechim": "Nominal -> OneHot, Ordinal -> .map()/OrdinalEncoder"},
    {"Muammo": "Nominalga raqam berish", "Yechim": "Sun'iy tartib yaratadi — QILMANG"},
    {"Muammo": "Yangi kategoriya kelishi", "Yechim": "handle_unknown='ignore'"},
    {"Muammo": "Bo'sh qiymat", "Yechim": "Mediana/moda/alohida toifa — bo'shlik ma'nosiga qarab"},
    {"Muammo": "Har xil shkala", "Yechim": "StandardScaler; daraxtlarga kerak emas"},
    {"Muammo": "Deyarli bir xil qatorlar", "Yechim": "Avval normallashtir, keyin drop_duplicates"},
    {"Muammo": "Hammasini bog'lash", "Yechim": "ColumnTransformer + Pipeline, remainder='drop'"},
    {"Muammo": "Uch oydan keyin eslash", "Yechim": "Dataset pasporti + md5 xesh"},
]))