Boosting: XGBoost, LightGBM, CatBoost

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.tree import DecisionTreeRegressor
from sklearn.metrics import root_mean_squared_error

df = pd.read_csv(DATA / "uy_ijara.csv")
son = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel"]
X, y = df[son], df["narx_usd"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42)

# 1-qadam: oddiy bashorat — hammaga o'rtacha
bashorat_tr = np.full(len(y_tr), y_tr.mean())
bashorat_te = np.full(len(y_te), y_tr.mean())
daraxtlar = []
lr = 0.1

print(f"{'qadam':>6} {'train RMSE':>12} {'test RMSE':>11}")
print("-" * 32)
print(f"{0:>6} {root_mean_squared_error(y_tr, bashorat_tr):>12.1f} "
      f"{root_mean_squared_error(y_te, bashorat_te):>11.1f}")

for qadam in range(1, 101):
    qoldiq = y_tr - bashorat_tr                       # HOZIRGI xato
    d = DecisionTreeRegressor(max_depth=3, random_state=42).fit(X_tr, qoldiq)
    daraxtlar.append(d)
    bashorat_tr = bashorat_tr + lr * d.predict(X_tr)  # xatoni bir oz tuzatamiz
    bashorat_te = bashorat_te + lr * d.predict(X_te)
    if qadam in (1, 5, 20, 50, 100):
        print(f"{qadam:>6} {root_mean_squared_error(y_tr, bashorat_tr):>12.1f} "
              f"{root_mean_squared_error(y_te, bashorat_te):>11.1f}")

from sklearn.ensemble import GradientBoostingRegressor

sk = GradientBoostingRegressor(n_estimators=100, learning_rate=0.1,
                               max_depth=3, random_state=42).fit(X_tr, y_tr)
print(f"Bizning qo'lda yozilgan: test RMSE = {root_mean_squared_error(y_te, bashorat_te):.1f}")
print(f"sklearn GradientBoosting: test RMSE = {root_mean_squared_error(y_te, sk.predict(X_te)):.1f}")

show(pd.DataFrame([
    {"Kutubxona": "XGBoost", "Yili": 2014, "Asosiy kuchi": "Birinchi mashhur; regularizatsiya ichida",
     "Kamchiligi": "Kategoriyani o'zi eplamaydi"},
    {"Kutubxona": "LightGBM", "Yili": 2017, "Asosiy kuchi": "ENG TEZ; katta ma'lumotda ustun",
     "Kamchiligi": "Kichik ma'lumotda overfitting ga moyil"},
    {"Kutubxona": "CatBoost", "Yili": 2017, "Asosiy kuchi": "Kategoriyani O'ZI eplaydi; sukut sozlamalari yaxshi",
     "Kamchiligi": "Sekinroq o'qitadi"},
]))

import time
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
from sklearn.ensemble import RandomForestRegressor

print(f"{'model':<26} {'CV R2':>9} {'±':>8} {'vaqt':>8}")
print("-" * 54)
for nom, m in [
    ("RandomForest (18-bob)", RandomForestRegressor(n_estimators=200, random_state=42)),
    ("GradientBoosting", GradientBoostingRegressor(random_state=42)),
    ("XGBoost", XGBRegressor(n_estimators=200, verbosity=0, random_state=42)),
    ("LightGBM", LGBMRegressor(n_estimators=200, verbose=-1, random_state=42)),
    ("CatBoost", CatBoostRegressor(iterations=200, verbose=0, random_seed=42)),
]:
    t0 = time.perf_counter()
    b = cross_val_score(m, X, y, cv=5, scoring="r2")
    dt = time.perf_counter() - t0
    print(f"{nom:<26} {b.mean():>9.4f} {b.std():>8.4f} {dt:>7.1f}s")

from xgboost import XGBRegressor
print(f"{'learning_rate':>14} {'n_estimators':>13} {'CV R2':>9}")
print("-" * 40)
for lr_ in [0.01, 0.05, 0.1, 0.3, 1.0]:
    b = cross_val_score(XGBRegressor(n_estimators=200, learning_rate=lr_, verbosity=0,
                                     random_state=42), X, y, cv=5, scoring="r2")
    print(f"{lr_:>14} {200:>13} {b.mean():>9.4f}")

print("Kichik learning_rate + ko'p daraxt:")
print(f"{'lr':>6} {'daraxtlar':>11} {'CV R2':>9}")
print("-" * 28)
for lr_, n in [(0.3, 50), (0.1, 150), (0.05, 300), (0.01, 1500)]:
    b = cross_val_score(XGBRegressor(n_estimators=n, learning_rate=lr_, verbosity=0,
                                     random_state=42), X, y, cv=5, scoring="r2")
    print(f"{lr_:>6} {n:>11} {b.mean():>9.4f}")

X_tr2, X_val, y_tr2, y_val = train_test_split(X_tr, y_tr, test_size=0.25, random_state=42)
m_es = XGBRegressor(n_estimators=3000, learning_rate=0.05, early_stopping_rounds=50,
                    verbosity=0, random_state=42)
m_es.fit(X_tr2, y_tr2, eval_set=[(X_val, y_val)], verbose=False)
print(f"Ruxsat berilgan daraxtlar : 3000")
print(f"Early stopping to'xtatdi  : {m_es.best_iteration} daraxtda")
print(f"Test R2                   : {m_es.score(X_te, y_te):.4f}")

from catboost import CatBoostClassifier
yc = df["ijaraga_berildi_30k"]
Xk = df[son + ["tuman"]]

# CatBoost matn ustunni TO'G'RIDAN-TO'G'RI qabul qiladi
cb = CatBoostClassifier(iterations=200, verbose=0, random_seed=42,
                        cat_features=["tuman"]).fit(Xk, yc)
print("CatBoost matn ustunni encoding SIZ qabul qildi — xato yo'q")
print(f"  train accuracy: {cb.score(Xk, yc):.4f}")
print()
# boshqalari uchun OneHot kerak
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder
from xgboost import XGBClassifier
q = Pipeline([
    ("t", ColumnTransformer([("m", OneHotEncoder(handle_unknown="ignore"), ["tuman"])],
                            remainder="passthrough")),
    ("m", XGBClassifier(n_estimators=200, verbosity=0, random_state=42)),
])
from sklearn.metrics import f1_score
Xk_tr, Xk_te, yk_tr, yk_te = train_test_split(Xk, yc, test_size=0.3, random_state=42, stratify=yc)

q.fit(Xk_tr, yk_tr)
print(f"XGBoost + OneHot:  test F1 = {f1_score(yk_te, q.predict(Xk_te)):.4f}")

cb2 = CatBoostClassifier(iterations=200, verbose=0, random_seed=42,
                         cat_features=["tuman"]).fit(Xk_tr, yk_tr)
print(f"CatBoost (o'zi)  :  test F1 = {f1_score(yk_te, cb2.predict(Xk_te)):.4f}")

show(pd.DataFrame([
    {"Parametr": "learning_rate", "Ma'nosi": "Har daraxt qancha tuzatadi", "Amaliy": "0.01-0.1"},
    {"Parametr": "n_estimators", "Ma'nosi": "Daraxtlar soni", "Amaliy": "early_stopping bilan topiladi"},
    {"Parametr": "max_depth", "Ma'nosi": "Daraxt chuqurligi", "Amaliy": "3-8 (RF dan SAYOZ)"},
    {"Parametr": "subsample", "Ma'nosi": "Har daraxt uchun qatorlar ulushi", "Amaliy": "0.7-1.0"},
    {"Parametr": "colsample_bytree", "Ma'nosi": "Har daraxt uchun ustunlar ulushi", "Amaliy": "0.7-1.0"},
    {"Parametr": "reg_lambda", "Ma'nosi": "L2 jarima (16-bob)", "Amaliy": "1-10"},
]))

show(pd.DataFrame([
    {"Tushuncha": "Boosting g'oyasi", "Bitta jumla": "Har daraxt oldingilarning XATOSINI o'rganadi"},
    {"Tushuncha": "Uch qator kod", "Bitta jumla": "qoldiq -> fit(X, qoldiq) -> bashorat += lr * yangi"},
    {"Tushuncha": "Daraxt chuqurligi", "Bitta jumla": "Sayoz (3-6) — RF dagidan farqli"},
    {"Tushuncha": "learning_rate", "Bitta jumla": "Kichik bo'lsa ko'proq daraxt kerak"},
    {"Tushuncha": "n_estimators", "Bitta jumla": "Oshirish OVERFITTING qiladi — RF dan farqli"},
    {"Tushuncha": "early_stopping", "Bitta jumla": "Validation to'xtaganda o'qitishni tugatadi"},
    {"Tushuncha": "XGBoost 2.0", "Bitta jumla": "early_stopping_rounds konstruktorga ko'chdi"},
    {"Tushuncha": "CatBoost", "Bitta jumla": "cat_features bilan matn ustunni o'zi eplaydi"},
]))