Mashq 2: iflos savdo ma'lumoti

import pandas as pd

sv = pd.read_csv(DATA / "iflos_savdo.csv")
print("shakli:", sv.shape)
print()
print(sv.dtypes.to_string())
print()
show(sv.head(8))

print("1) Sana formatlari:")
nuqtali = sv.order_date.astype(str).str.match(r"\d{2}\.\d{2}\.\d{4}").sum()
iso = sv.order_date.astype(str).str.match(r"\d{4}-\d{2}-\d{2}").sum()
print(f"   nuqtali (26.02.2025): {nuqtali}")
print(f"   ISO     (2025-05-26): {iso}")
print(f"   boshqa/bo'sh        : {len(sv) - nuqtali - iso}")
print()
print("2) unit_price turi:", sv.unit_price.dtype)
print("   bo'shliq bilan yozilgan:", sv.unit_price.astype(str).str.contains(" ").sum(), "ta")
print("   namunalar:", sv.unit_price.astype(str).head(4).tolist())
print()
print("3) region — nechta xil qiymat:", sv.region.nunique())
print("   lekin haqiqiy viloyatlar soni ancha kam. Mana nega:")
for v in sorted(sv.region.dropna().unique())[:9]:
    print("     ", repr(v))
print("     ... jami", sv.region.nunique(), "xil yozilish")
print()
print("4) Takrorlangan qatorlar:", int(sv.duplicated().sum()))
print()
print("5) Bo'sh qiymatlar:")
yoq = sv.isna().sum()
print(yoq[yoq > 0].to_string())
print()
print("6) status (bo'lajak target):")
print(sv.status.value_counts(dropna=False).to_string())

import pandas as pd
show(pd.DataFrame([
    {"#": 1, "Yozib qo'yiladigan javob": "Nechta qator, nechta ustun"},
    {"#": 2, "Yozib qo'yiladigan javob": "\"Men ___ ni ___ dan bashorat qilaman\" — bitta jumla"},
    {"#": 3, "Yozib qo'yiladigan javob": "Vazifa turi: regression / binary / multiclass"},
    {"#": 4, "Yozib qo'yiladigan javob": "Target nisbati va BAZA raqami"},
    {"#": 5, "Yozib qo'yiladigan javob": "Split qildingizmi? stratify kerakmi?"},
    {"#": 6, "Yozib qo'yiladigan javob": "Dummy natijasi"},
    {"#": 7, "Yozib qo'yiladigan javob": "Qaysi ustunlarni tashladingiz va NEGA"},
    {"#": 8, "Yozib qo'yiladigan javob": "Missing va turlar bilan nima qildingiz"},
    {"#": 9, "Yozib qo'yiladigan javob": "Birinchi halol natijangiz"},
    {"#": 10, "Yozib qo'yiladigan javob": "Qaysi metrikani tanladingiz va NEGA"},
    {"#": 11, "Yozib qo'yiladigan javob": "Xatolar qaysi guruhda to'plangan"},
    {"#": 12, "Yozib qo'yiladigan javob": "Qaysi BITTA o'zgarishni qildingiz va natija"},
]))

show(pd.DataFrame([
    {"T": 1, "Tozalash vazifasi": "order_date ni bitta formatga keltirish (ikki format bor)"},
    {"T": 2, "Tozalash vazifasi": "unit_price ni songa aylantirish (bo'shliqni olib tashlash)"},
    {"T": 3, "Tozalash vazifasi": "region ni normallashtirish — registr, bo'shliq, KIRILL alifbosi"},
    {"T": 4, "Tozalash vazifasi": "Takrorlangan qatorlarni olib tashlash"},
    {"T": 5, "Tozalash vazifasi": "Bo'sh status bilan nima qilish (u sizning target ingiz)"},
    {"T": 6, "Tozalash vazifasi": "Target yaratish: nimani bashorat qilasiz?"},
]))

import numpy as np

KIRILL = {"андижон": "andijon", "бухоро": "buxoro", "самарқанд": "samarqand",
          "тошкент шаҳри": "toshkent shahri", "фарғона": "fargona"}

def normallashtir(x):
    if pd.isna(x):
        return np.nan
    s = str(x).strip().lower()
    s = KIRILL.get(s, s)
    return s.replace("'", "").replace("`", "").replace("ʻ", "")

toza = sv.drop_duplicates().copy()
toza["region_toza"] = toza.region.map(normallashtir)
toza["narx"] = pd.to_numeric(
    toza.unit_price.astype(str).str.replace(r"[^\d.]", "", regex=True), errors="coerce")
toza["sana"] = pd.to_datetime(toza.order_date, format="mixed", dayfirst=True, errors="coerce")

print(f"qatorlar      : {len(sv)} -> {len(toza)}  (dublikat olib tashlandi)")
print(f"region xilligi: {sv.region.nunique()} -> {toza.region_toza.nunique()}")
print(f"narx turi     : {sv.unit_price.dtype} -> {toza.narx.dtype}")
print(f"narx NaN      : {int(toza.narx.isna().sum())}")
print(f"sana NaN      : {int(toza.sana.isna().sum())}")
print()
print("Normallashtirilgandan keyingi viloyatlar:")
print(toza.region_toza.value_counts().to_string())

from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.dummy import DummyClassifier
from sklearn.metrics import accuracy_score, f1_score, average_precision_score

m = toza.dropna(subset=["status", "narx", "sana"]).copy()
m["bekor"] = (m.status == "cancelled").astype(int)
m["oy"] = m.sana.dt.month
m["hafta_kuni"] = m.sana.dt.dayofweek
m["quantity"] = m.quantity.fillna(m.quantity.median())
m["tushum"] = m.quantity * m.narx

son = ["narx", "quantity", "tushum", "oy", "hafta_kuni"]
matn = ["region_toza", "category"]
X, y = m[son + matn], m["bekor"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42, stratify=y)

print(f"Model uchun qatorlar: {len(m)}")
print(f"Target 'bekor' ulushi: {y.mean():.2%}   <-- qattiq nomutanosib")
print()
baza = accuracy_score(y_te, DummyClassifier(strategy="most_frequent").fit(X_tr[son], y_tr).predict(X_te[son]))
print(f"BAZA accuracy: {baza:.4f}   <-- accuracy bu yerda FOYDASIZ")

quvur = Pipeline([
    ("tayyorla", ColumnTransformer([
        ("son", Pipeline([("toldir", SimpleImputer(strategy="median")),
                          ("shkala", StandardScaler())]), son),
        ("matn", Pipeline([("toldir", SimpleImputer(strategy="most_frequent")),
                           ("kod", OneHotEncoder(handle_unknown="ignore"))]), matn),
    ])),
    ("model", LogisticRegression(max_iter=3000, class_weight="balanced")),
]).fit(X_tr, y_tr)

b = quvur.predict(X_te)
e = quvur.predict_proba(X_te)[:, 1]
print(f"Accuracy          : {accuracy_score(y_te, b):.4f}  (baza {baza:.4f})")
print(f"F1 (bekor sinfi)  : {f1_score(y_te, b, zero_division=0):.4f}")
print(f"Average Precision : {average_precision_score(y_te, e):.4f}")
print(f"AP bazasi         : {y_te.mean():.4f}  (= pozitiv ulushi)")