Mốc 1: pipeline dữ liệu và đặc trưng có kiểm thử

from anhti_eta.config import load_settings
from anhti_eta.data import clean, load_raw, time_split
from anhti_eta.features import build_features

settings = load_settings()
raw = load_raw(settings.raw_path)
df, report = clean(raw, settings.late_margin_minutes)
for key, value in report.items():
    print(f"{key:26} {value}")

train, val, test = time_split(df, settings.val_start, settings.test_start)
for name, part in [("train", train), ("val", val), ("test", test)]:
    print(f"{name:5} rows={len(part):6} from {part['created_at'].min():%Y-%m-%d} to {part['created_at'].max():%Y-%m-%d}"
          f" rain={part['is_rain'].mean():.3f} late_30={part['is_late_30'].mean():.3f}")

X = build_features(train)
print("feature shape:", X.shape)
print(X.iloc[0].to_string())

rows_in                    50500
duplicates_removed         500
district_fixed             3434
district_unknown_dropped   0
distance_negative_fixed    271
distance_meters_fixed      184
weight_missing_kept        1534
rows_out                   50000
train rows= 33312 from 2026-01-01 to 2026-04-30 rain=0.109 late_30=0.051
val   rows=  8487 from 2026-05-01 to 2026-05-31 rain=0.423 late_30=0.065
test  rows=  8201 from 2026-06-01 to 2026-06-30 rain=0.636 late_30=0.067
feature shape: (33312, 14)
distance_km               1.53
items                      5.0
weight_kg                 4.15
order_value_vnd       454000.0
shipper_exp_months        15.0
is_rain                    0.0
hour                       7.0
weekday                    3.0
is_peak                    0.0
is_weekend                 0.0
district               Xom Vui
payment                    cod
service                express
vehicle                   bike

def leaky_features(frame):
    X = build_features(frame)
    # sai: trung bình theo khách tính trên mọi đơn, kể cả chính đơn này và các đơn tương lai
    X["customer_mean_minutes"] = frame.groupby("customer_id")["delivery_minutes"].transform("mean")
    return X

def train_rows_changed_by_future(builder):
    future = df["created_at"] >= settings.val_start
    edited = df.assign(delivery_minutes=df["delivery_minutes"].where(~future, df["delivery_minutes"] + 100))
    before, after = builder(df).iloc[:, -1].fillna(-1), builder(edited).iloc[:, -1].fillna(-1)
    return int((before != after)[~future].sum()), int((~future).sum())

ridge, order-time features only : val MAE 10.46
ridge, + leaky customer mean    : val MAE 10.30
ridge, + past-only customer mean: val MAE 10.46
train rows whose customer_mean_minutes changes when May-June delivery times change:
  leaky builder    : 28904 of 33312
  past-only builder: 0 of 33312

import os
import tomllib
from dataclasses import dataclass, fields, replace
from pathlib import Path

PROJECT_DIR = Path(__file__).resolve().parents[1]
DEFAULT_DATA_DIR = Path(__file__).resolve().parents[3] / "data"


@dataclass(frozen=True)
class Settings:
    data_dir: Path = DEFAULT_DATA_DIR
    artifacts_dir: Path = PROJECT_DIR / "artifacts"
    raw_file: str = "orders_raw.csv"
    val_start: str = "2026-05-01"
    test_start: str = "2026-06-01"
    late_margin_minutes: int = 30
    seed: int = 42
    cost_call_vnd: int = 4_000
    cost_missed_vnd: int = 60_000
    bootstrap_rounds: int = 1_000

    @property
    def raw_path(self) -> Path:
        return self.data_dir / self.raw_file

    @property
    def model_path(self) -> Path:
        return self.artifacts_dir / "model.joblib"


def load_settings(path: str | Path | None = None, **overrides) -> Settings:
    values: dict = {}
    if path is not None and Path(path).exists():
        values.update(tomllib.loads(Path(path).read_text(encoding="utf-8")))
    if "ANHTI_DATA" in os.environ:
        values["data_dir"] = os.environ["ANHTI_DATA"]
    if "ANHTI_ARTIFACTS" in os.environ:
        values["artifacts_dir"] = os.environ["ANHTI_ARTIFACTS"]
    values.update(overrides)
    known = {f.name for f in fields(Settings)}
    unknown = set(values) - known
    if unknown:
        raise ValueError(f"unknown settings: {sorted(unknown)}")
    for key in ("data_dir", "artifacts_dir"):
        if key in values:
            values[key] = Path(values[key])
    return replace(Settings(), **values)

from pathlib import Path

import numpy as np
import pandas as pd

DISTRICTS = ("Dau Pho", "Xom Vui", "Cho Moi", "Ben Song", "Doi Thong", "Cau Go", "Bai Bien", "Khu Cong Nghiep")
PROMISED_MINUTES = {"express": 60, "standard": 120}
MAX_DISTANCE_KM = 100.0


def load_raw(path: str | Path) -> pd.DataFrame:
    raw = pd.read_csv(path)
    raw["created_at"] = pd.to_datetime(raw["created_at"], format="%Y-%m-%d %H:%M:%S")
    raw["note"] = raw["note"].fillna("")
    return raw


def clean(raw: pd.DataFrame, late_margin_minutes: int = 30) -> tuple[pd.DataFrame, dict]:
    df = raw.copy()
    report = {"rows_in": len(df)}
    # TODO 1: bỏ dòng trùng order_id (giữ dòng đầu), ghi report["duplicates_removed"].
    # TODO 2: chuẩn hóa district (bỏ khoảng trắng, đưa về đúng tên trong DISTRICTS), ghi report["district_fixed"]
    #         (số dòng có giá trị bị đổi) và report["district_unknown_dropped"] (số dòng không nhận ra, bị bỏ).
    # TODO 3: distance_km âm thì lấy trị tuyệt đối (report["distance_negative_fixed"]); lớn hơn MAX_DISTANCE_KM
    #         là mét gõ nhầm thành km, chia 1000 (report["distance_meters_fixed"]); làm tròn 2 chữ số.
    # TODO 4: giữ weight_kg thiếu là NaN, đếm vào report["weight_missing_kept"].
    # TODO 5: thêm promised_minutes theo service và is_late_30 = delivery_minutes > promised_minutes + late_margin_minutes.
    # TODO 6: sắp xếp theo created_at rồi order_id, reset_index, ghi report["rows_out"].
    raise NotImplementedError


def time_split(df: pd.DataFrame, val_start: str, test_start: str) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
    # TODO 7: báo ValueError nếu val_start không trước test_start.
    # train: created_at < val_start; val: từ val_start tới trước test_start; test: từ test_start trở đi.
    raise NotImplementedError

import pandas as pd

from .data import DISTRICTS

CATEGORIES = {
    "district": list(DISTRICTS),
    "payment": ["cod", "prepaid"],
    "service": ["express", "standard"],
    "vehicle": ["bike", "motorbike", "van"],
}
INPUT_COLUMNS = ["created_at", "district", "distance_km", "items", "weight_kg", "order_value_vnd",
                 "payment", "service", "vehicle", "shipper_exp_months", "is_rain"]
NUMERIC = ["distance_km", "items", "weight_kg", "order_value_vnd", "shipper_exp_months", "is_rain",
           "hour", "weekday", "is_peak", "is_weekend"]
CATEGORICAL = list(CATEGORIES)
FEATURES = NUMERIC + CATEGORICAL
TARGET_COLUMNS = ("delivery_minutes", "is_late", "is_late_30")
DTYPES = {col: pd.CategoricalDtype(cats) for col, cats in CATEGORIES.items()}


def build_features(df: pd.DataFrame) -> pd.DataFrame:
    # TODO 8: báo KeyError nếu thiếu cột trong INPUT_COLUMNS, rồi chỉ đọc các cột đó.
    # TODO 9: cột số đổi sang float; is_rain thành 0.0 hoặc 1.0.
    # TODO 10: từ created_at tạo hour, weekday (0 là thứ Hai), is_peak (11-12h hoặc 17-19h), is_weekend.
    # TODO 11: cột phân loại thành pd.Categorical với dtype cố định trong DTYPES (giá trị lạ thành NaN).
    # Trả về DataFrame có đúng thứ tự cột FEATURES.
    raise NotImplementedError

import pandas as pd
import pytest

from anhti_eta.config import load_settings
from anhti_eta.data import clean, load_raw, time_split


def tiny_raw() -> pd.DataFrame:
    return pd.DataFrame({
        "order_id": ["A1", "A2", "A2", "A3", "A4", "A5"],
        "created_at": pd.to_datetime(["2026-04-30 11:05", "2026-05-02 08:00", "2026-05-02 08:00",
                                      "2026-06-06 18:30", "2026-01-03 09:00", "2026-06-01 00:00"]),
        "customer_id": ["C1", "C2", "C2", "C3", "C4", "C5"],
        "district": ["Xom Vui", "XOM VUI", "XOM VUI", "Cho Moi ", "Dau Pho", "Bai Bien"],
        "distance_km": [2.5, -3.1, -3.1, 4200.0, 1.0, 7.25],
        "items": [1, 2, 2, 3, 1, 4],
        "weight_kg": [1.2, None, None, 3.0, 0.5, 2.0],
        "order_value_vnd": [100000, 200000, 200000, 300000, 50000, 120000],
        "payment": ["cod", "prepaid", "prepaid", "cod", "cod", "prepaid"],
        "service": ["express", "standard", "standard", "express", "standard", "express"],
        "vehicle": ["bike", "motorbike", "motorbike", "van", "bike", "motorbike"],
        "shipper_exp_months": [3, 12, 12, 30, 0, 8],
        "is_rain": [False, True, True, True, False, False],
        "note": ["", "giao gap", "giao gap", "", "", ""],
        "promised_minutes": [60, 120, 120, 60, 120, 60],
        "delivery_minutes": [95.0, 150.0, 150.0, 85.0, 40.0, 91.0],
        "is_late": [True, True, True, True, False, True],
    })


@pytest.fixture
def raw_small() -> pd.DataFrame:
    return tiny_raw()


@pytest.fixture(scope="session")
def settings():
    return load_settings()


@pytest.fixture(scope="session")
def splits(settings):
    df, _ = clean(load_raw(settings.raw_path), settings.late_margin_minutes)
    return time_split(df, settings.val_start, settings.test_start)


@pytest.fixture(scope="session")
def bundle(splits, settings):
    from anhti_eta.train import fit_final  # chỉ cần từ mốc 2 trở đi

    train, val, _ = splits
    b = fit_final(train, val, settings, "ridge", "logistic")
    b["baseline_minutes"] = float(train["delivery_minutes"].median())
    b["version"] = "eta-test"
    return b

import pandas as pd
import pytest

from anhti_eta.data import DISTRICTS, clean, time_split


def test_clean_fixes_known_issues(raw_small):
    df, report = clean(raw_small)
    assert len(df) == 5
    assert report["duplicates_removed"] == 1
    assert report["district_fixed"] == 2
    assert report["distance_negative_fixed"] == 1
    assert report["distance_meters_fixed"] == 1
    assert report["weight_missing_kept"] == 1
    assert set(df["district"]) <= set(DISTRICTS)
    by_id = df.set_index("order_id")
    assert by_id.loc["A2", "distance_km"] == pytest.approx(3.1)
    assert by_id.loc["A3", "distance_km"] == pytest.approx(4.2)
    assert pd.isna(by_id.loc["A2", "weight_kg"])


def test_clean_does_not_modify_input(raw_small):
    before = raw_small.copy()
    clean(raw_small)
    pd.testing.assert_frame_equal(raw_small, before)


def test_late_30_uses_promise_plus_margin(raw_small):
    df, _ = clean(raw_small, late_margin_minutes=30)
    flags = df.set_index("order_id")["is_late_30"].to_dict()
    assert flags == {"A1": True, "A2": False, "A3": False, "A4": False, "A5": True}


def test_clean_output_sorted_by_time(raw_small):
    df, _ = clean(raw_small)
    assert df["created_at"].is_monotonic_increasing


def test_time_split_by_date_boundaries(raw_small):
    df, _ = clean(raw_small)
    train, val, test = time_split(df, "2026-05-01", "2026-06-01")
    assert list(train["order_id"]) == ["A4", "A1"]
    assert list(val["order_id"]) == ["A2"]
    assert list(test["order_id"]) == ["A5", "A3"]


def test_time_split_rejects_reversed_dates(raw_small):
    df, _ = clean(raw_small)
    with pytest.raises(ValueError):
        time_split(df, "2026-06-01", "2026-05-01")


def test_real_data_split_has_no_time_overlap(splits):
    train, val, test = splits
    assert len(train) + len(val) + len(test) == 50_000
    assert train["created_at"].max() < val["created_at"].min()
    assert val["created_at"].max() < test["created_at"].min()


def test_real_raw_cleans_back_to_reference_distances(splits, settings):
    cleaned = pd.concat(splits)
    reference = pd.read_parquet(settings.data_dir / "orders.parquet")
    merged = cleaned.merge(reference, on="order_id", suffixes=("", "_ref"))
    assert len(merged) == 50_000
    assert (merged["distance_km"] - merged["distance_km_ref"]).abs().max() < 1e-9
    assert (merged["district"] == merged["district_ref"]).all()

import numpy as np
import pandas as pd

from anhti_eta.data import clean
from anhti_eta.features import FEATURES, TARGET_COLUMNS, build_features


def test_features_never_include_targets():
    assert not set(FEATURES) & set(TARGET_COLUMNS)
    assert "promised_minutes" not in FEATURES


def test_features_ignore_target_values(raw_small):
    df, _ = clean(raw_small)
    scrambled = df.copy()
    scrambled["delivery_minutes"] = 999.0
    scrambled["is_late"] = ~scrambled["is_late"]
    scrambled["is_late_30"] = ~scrambled["is_late_30"]
    pd.testing.assert_frame_equal(build_features(df), build_features(scrambled))


def test_features_of_past_orders_ignore_future_orders(splits):
    train, val, _ = splits
    window = pd.concat([train.tail(300), val.head(300)]).reset_index(drop=True)
    past_only = build_features(window.head(300))
    with_future = build_features(window).head(300)
    pd.testing.assert_frame_equal(past_only, with_future)


def test_time_features_from_created_at(raw_small):
    df, _ = clean(raw_small)
    X = build_features(df).set_index(df["order_id"])
    assert X.loc["A1", "hour"] == 11 and X.loc["A1", "is_peak"] == 1
    assert X.loc["A3", "hour"] == 18 and X.loc["A3", "is_peak"] == 1
    assert X.loc["A3", "is_weekend"] == 1
    assert X.loc["A4", "is_peak"] == 0


def test_unknown_category_becomes_missing(raw_small):
    df, _ = clean(raw_small)
    df.loc[0, "vehicle"] = "drone"
    X = build_features(df)
    assert pd.isna(X.loc[0, "vehicle"])
    assert list(X.columns) == FEATURES
    assert np.isnan(X.loc[X.index[df["order_id"] == "A2"][0], "weight_kg"])

cd m01/project-eta
PYTHONPATH=starter uv run pytest -q tests/test_data.py tests/test_features.py