Ma'lumot formatlari: CSV dan Parquet gacha

901234567
907654321

import pandas as pd
import io

xom = "id,kod,narx,sana,faol\n1,007,10.5,2026-09-01,true\n2,042,20.0,2026-09-02,false\n"
d = pd.read_csv(io.StringIO(xom))
print("CSV dan o'qilgandan keyingi turlar:")
for ustun, tur in d.dtypes.items():
    print(f"  {ustun:8} {str(tur):10} namuna: {d[ustun].iloc[0]!r}")

diagramma(
    {
        "CSV (qator bo'yicha)": (0, 0, 4.4, 0.7, "manba"),
        "1-qator: Aziz | A | 85": (0, 0.9, 4.4, 0.6, "oddiy"),
        "2-qator: Bobur | A | 92": (0, 1.7, 4.4, 0.6, "oddiy"),
        "3-qator: Dilnoza | B | 78": (0, 2.5, 4.4, 0.6, "oddiy"),
        "Parquet (ustun bo'yicha)": (5.6, 0, 4.4, 0.7, "saqlash"),
        "ism: Aziz, Bobur, Dilnoza": (5.6, 0.9, 4.4, 0.6, "oddiy"),
        "guruh: A, A, B": (5.6, 1.7, 4.4, 0.6, "oddiy"),
        "baho: 85, 92, 78": (5.6, 2.5, 4.4, 0.6, "oddiy"),
    },
    nom="qator_ustun.png", kenglik=13,
    izoh="Chapda har qator birga yotadi. O'ngda har ustun birga yotadi — bitta ustunni o'qish uchun qolganini o'qish shart emas.",
)

import pandas as pd
from pathlib import Path

ISH.mkdir(parents=True, exist_ok=True)
d = pd.read_csv(DATA / "ijara.csv")
print(f"ma'lumot: {len(d):,} qator, {len(d.columns)} ustun")

yollar = {}
d.to_csv(ISH / "t.csv", index=False)
yollar["CSV"] = ISH / "t.csv"

import gzip, shutil
with open(ISH / "t.csv", "rb") as f_in, gzip.open(ISH / "t.csv.gz", "wb", compresslevel=6) as f_out:
    shutil.copyfileobj(f_in, f_out)
yollar["CSV + gzip"] = ISH / "t.csv.gz"

for siqish in ["snappy", "gzip", "zstd"]:
    try:
        yol = ISH / f"t_{siqish}.parquet"
        d.to_parquet(yol, compression=siqish, index=False)
        yollar[f"Parquet ({siqish})"] = yol
    except Exception as e:
        print(f"  {siqish}: {type(e).__name__}")

asos = yollar["CSV"].stat().st_size
qatorlar = []
for nom, yol in yollar.items():
    hajm = yol.stat().st_size
    qatorlar.append({"format": nom, "hajm, KB": round(hajm / 1024, 1),
                     "CSV ga nisbatan": f"{hajm / asos:.2f}x",
                     "tejaldi": f"{100 * (1 - hajm / asos):.0f}%"})
show(pd.DataFrame(qatorlar))

import time

def olch(f, n=5):
    f()
    t0 = time.perf_counter()
    for _ in range(n):
        natija = f()
    return (time.perf_counter() - t0) / n * 1000, natija

qatorlar = []
ms, _ = olch(lambda: pd.read_csv(ISH / "t.csv"))
qatorlar.append({"amal": "CSV — butun fayl", "ms": round(ms, 1)})

ms, _ = olch(lambda: pd.read_parquet(ISH / "t_snappy.parquet"))
qatorlar.append({"amal": "Parquet — butun fayl", "ms": round(ms, 1)})

ms, _ = olch(lambda: pd.read_csv(ISH / "t.csv", usecols=["narx_mln", "broker"]))
qatorlar.append({"amal": "CSV — 2 ustun", "ms": round(ms, 1)})

ms, _ = olch(lambda: pd.read_parquet(ISH / "t_snappy.parquet", columns=["narx_mln", "broker"]))
qatorlar.append({"amal": "Parquet — 2 ustun", "ms": round(ms, 1)})
show(pd.DataFrame(qatorlar))

import pandas as pd
from datetime import date

sinov = pd.DataFrame({
    "kod": ["007", "042", "100"],
    "sana": pd.to_datetime(["2026-09-01", "2026-09-02", "2026-09-03"]),
    "narx": [10.5, 20.0, 30.25],
    "faol": [True, False, True],
    "toifa": pd.Categorical(["A", "B", "A"]),
})
print("ASL turlar:")
for u, t in sinov.dtypes.items():
    print(f"  {u:7} {t}")

sinov.to_csv(ISH / "turlar.csv", index=False)
sinov.to_parquet(ISH / "turlar.parquet", index=False)
csv_qayta = pd.read_csv(ISH / "turlar.csv")
pq_qayta = pd.read_parquet(ISH / "turlar.parquet")

qatorlar = []
for u in sinov.columns:
    qatorlar.append({
        "ustun": u,
        "asl": str(sinov[u].dtype),
        "CSV dan keyin": str(csv_qayta[u].dtype),
        "Parquet dan keyin": str(pq_qayta[u].dtype),
        "CSV saqladimi": str(sinov[u].dtype) == str(csv_qayta[u].dtype),
    })
show(pd.DataFrame(qatorlar))
print("\nCSV dan keyin 'kod' birinchi qiymati:", repr(csv_qayta["kod"].iloc[0]))
print("Parquet dan keyin                    :", repr(pq_qayta["kod"].iloc[0]))

import pyarrow.parquet as pq

fayl = pq.ParquetFile(ISH / "t_snappy.parquet")
print("sxema (birinchi 6 ustun):")
for maydon in list(fayl.schema_arrow)[:6]:
    print(f"  {maydon.name:24} {maydon.type}")
print(f"\nqator guruhlari (row groups): {fayl.num_row_groups}")
print(f"jami qatorlar: {fayl.metadata.num_rows:,}")

# har ustun uchun statistika — so'rovni tezlashtiradi
guruh = fayl.metadata.row_group(0)
print("\nustun statistikasi (1-guruh):")
for i in range(min(4, guruh.num_columns)):
    ust = guruh.column(i)
    st = ust.statistics
    if st is not None:
        print(f"  {ust.path_in_schema:22} min={st.min}  max={st.max}  siqilgan={ust.total_compressed_size:>7} bayt")

import pandas as pd
show(pd.DataFrame([
    {"holat": "Odam Excel da ochadi", "format": "CSV", "sabab": "Universal, tahrirlash oson"},
    {"holat": "API javobi, ichma-ich tuzilish", "format": "JSON", "sabab": "Ierarxiya saqlanadi"},
    {"holat": "Ma'lumot ombori, tahlil", "format": "Parquet", "sabab": "Ustunli, siqilgan, turli"},
    {"holat": "Oqim (streaming) xabari", "format": "Avro / JSON", "sabab": "Qatorli, sxema evolyutsiyasi"},
    {"holat": "Model artefakti", "format": "joblib / ONNX", "sabab": "26-bobda batafsil"},
    {"holat": "Jadval versiyalari, time travel", "format": "Iceberg", "sabab": "19-bob"},
]))

import pandas as pd
qatorlar = []
for n in (100, 1_000, 10_000, 100_000):
    kichik = d.head(n) if n <= len(d) else pd.concat([d] * (n // len(d) + 1)).head(n)
    kichik.to_csv(ISH / "k.csv", index=False)
    kichik.to_parquet(ISH / "k.parquet", compression="snappy", index=False)
    c = (ISH / "k.csv").stat().st_size
    p = (ISH / "k.parquet").stat().st_size
    qatorlar.append({"qatorlar": f"{n:,}", "CSV, KB": round(c / 1024, 1),
                     "Parquet, KB": round(p / 1024, 1), "nisbat": f"{p / c:.2f}x",
                     "Parquet foydalimi": "ha" if p < c else "yo'q"})
show(pd.DataFrame(qatorlar))

"""Ma'lumotni o'qish va yozish — bitta joyda, format tanlovi bilan."""
from pathlib import Path

import pandas as pd

SIQISH = "snappy"


def yoz(df: pd.DataFrame, yol: str | Path) -> Path:
    """Kengaytmaga qarab format tanlaydi. Parquet uchun turlar saqlanadi."""
    p = Path(yol)
    p.parent.mkdir(parents=True, exist_ok=True)
    if p.suffix == ".parquet":
        df.to_parquet(p, compression=SIQISH, index=False)
    elif p.suffix == ".csv":
        df.to_csv(p, index=False, encoding="utf-8")
    else:
        raise ValueError(f"noma'lum format: {p.suffix} (.parquet yoki .csv kutilgan)")
    return p


def oqi(yol: str | Path, ustunlar: list[str] | None = None) -> pd.DataFrame:
    """Parquet bo'lsa faqat kerakli ustunlarni diskdan oladi."""
    p = Path(yol)
    if not p.exists():
        raise FileNotFoundError(f"ma'lumot topilmadi: {p}")
    if p.suffix == ".parquet":
        return pd.read_parquet(p, columns=ustunlar)
    if p.suffix == ".csv":
        return pd.read_csv(p, usecols=ustunlar)
    raise ValueError(f"noma'lum format: {p.suffix}")


def csv_dan_parquet(manba: str | Path, nishon: str | Path) -> dict[str, float]:
    """CSV ni Parquet ga o'tkazadi va tejamni qaytaradi."""
    df = pd.read_csv(manba)
    yoz(df, nishon)
    c = Path(manba).stat().st_size
    p = Path(nishon).stat().st_size
    return {"csv_kb": round(c / 1024, 1), "parquet_kb": round(p / 1024, 1),
            "tejam_foiz": round(100 * (1 - p / c), 1)}

python3 -c "
import sys; sys.path.insert(0, 'src')
from quvur.saqlash import csv_dan_parquet, oqi

for nom in ['ijara', 'ballon', 'buyurtma']:
    n = csv_dan_parquet(f'../datasets/{nom}.csv', f'../datasets/{nom}.parquet')
    print(f\"  {nom:10} {n['csv_kb']:>9.1f} KB -> {n['parquet_kb']:>8.1f} KB   tejam {n['tejam_foiz']:>5.1f}%\")

d = oqi('../datasets/ijara.parquet', ustunlar=['narx_mln', 'broker'])
print()
print('faqat 2 ustun o\'qildi:', list(d.columns), '|', len(d), 'qator')
"