from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_validate
pipe = Pipeline([
(“scaler”, StandardScaler()),
(“model”, LogisticRegression(max_iter=1000)),
])
result = cross_validate(pipe, X, y, cv=5, scoring=[“accuracy”, “f1_macro”])
print(result[“test_f1_macro”].mean())

预处理必须只从训练折学习，再应用到验证折。
如果你先对全数据 fit scaler，再交叉验证，验证分数就被污染了。

cd ~/Code/python-learning/week05-sklearn-baselines
source .venv/bin/activate.fish
python src/train_baselines.py

cd ~/Code/python-learning
mkdir -p week06-cross-validation-model-selection
cd week06-cross-validation-model-selection
uv init
uv venv
source .venv/bin/activate.fish
uv add pandas numpy scikit-learn tabulate
mkdir -p src reports data/raw data/processed
code .

week06-cross-validation-model-selection/
├── data/
│   ├── raw/
│   └── processed/
├── reports/
│   └── week06_cv_report.md
├── src/
│   └── model_selection.py
├── pyproject.toml
├── uv.lock
└── README.md

cross validation + pipeline + grid search + result table

train -> valid -> choose model -> test

把训练数据切成 K 份
第 1 次：第 1 份做验证，其余做训练
第 2 次：第 2 份做验证，其余做训练
...
第 K 次：第 K 份做验证，其余做训练
最后取 K 次分数的均值和标准差

from sklearn.model_selection import KFold
cv = KFold(n_splits=5, shuffle=True, random_state=42)

from sklearn.model_selection import StratifiedKFold
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

src/model_selection.py

from pathlib import Path
import pandas as pd
from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, roc_auc_score
from sklearn.model_selection import GridSearchCV, StratifiedKFold, train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.tree import DecisionTreeClassifier
RANDOM_STATE = 42
REPORT_PATH = Path(“reports/week06_cv_report.md”)
def load_data():
dataset = load_breast_cancer(as_frame=True)
return dataset.data, dataset.target, list(dataset.target_names)
def train_test_holdout(X, y):
return train_test_split(
X,
y,
test_size=0.2,
random_state=RANDOM_STATE,
stratify=y,
)
def build_search_spaces():
return {
“logistic_regression”: {
“pipeline”: Pipeline(
steps=[
(“scaler”, StandardScaler()),
(“model”, LogisticRegression(max_iter=3000, random_state=RANDOM_STATE)),
]
),
“params”: {
“model__C”: [0.1, 1.0, 10.0],
“model__class_weight”: [None, “balanced”],
},
},
“knn”: {
“pipeline”: Pipeline(
steps=[
(“scaler”, StandardScaler()),
(“model”, KNeighborsClassifier()),
]
),
“params”: {
“model__n_neighbors”: [3, 5, 9, 15],
“model__weights”: [“uniform”, “distance”],
},
},
“decision_tree”: {
“pipeline”: Pipeline(
steps=[
(“model”, DecisionTreeClassifier(random_state=RANDOM_STATE)),
]
),
“params”: {
“model__max_depth”: [2, 3, 4, 5, None],
“model__min_samples_leaf”: [1, 3, 5],
},
},
“random_forest”: {
“pipeline”: Pipeline(
steps=[
(“model”, RandomForestClassifier(random_state=RANDOM_STATE)),
]
),
“params”: {
“model__n_estimators”: [100, 200],
“model__max_depth”: [3, 5, None],
“model__min_samples_leaf”: [1, 3],
},
},
}
def evaluate_classifier(model, X_test, y_test):
predictions = model.predict(X_test)
probabilities = model.predict_proba(X_test)[:, 1]
return {
“test_accuracy”: accuracy_score(y_test, predictions),
“test_precision”: precision_score(y_test, predictions),
“test_recall”: recall_score(y_test, predictions),
“test_f1”: f1_score(y_test, predictions),
“test_roc_auc”: roc_auc_score(y_test, probabilities),
}
def run_model_selection(X_train, y_train):
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=RANDOM_STATE)
searches = []
plain for model_name, config in build_search_spaces().items():
search = GridSearchCV(
estimator=config[“pipeline”],
param_grid=config[“params”],
scoring=“f1”,
cv=cv,
n_jobs=-1,
refit=True,
return_train_score=True,
)
search.fit(X_train, y_train)
searches.append(
{
“model”: model_name,
“best_score”: search.best_score_,
“best_params”: search.best_params_,
“best_estimator”: search.best_estimator_,
}
)
plain return sorted(searches, key=lambda row: row[“best_score”], reverse=True)
def make_report(results, test_metrics, target_names):
table_rows = []
for row in results:
table_rows.append(
f”| {row[‘model’]} | {row[‘best_score’]:.4f} | {row['best_params']} |”
)
plain best = results[0]
report = f"""# Week 06 Cross Validation Report
Dataset

Source: sklearn breast cancer dataset
Task: binary classification
Target names:

Validation Design

Holdout test size: 20%
Cross validation: StratifiedKFold with 5 folds
Model selection metric: F1
Leakage prevention: preprocessing is inside sklearn Pipeline

Model Selection Results















Model
Mean CV F1
Best Params
{chr(10).join(table_rows)}

Best Model

Selected model: 
Selected by: highest mean cross-validation F1 on the training portion

Final Holdout Test Metrics





























Metric
Value
Accuracy
{test_metrics[‘test_accuracy’]:.4f}
Precision
{test_metrics[‘test_precision’]:.4f}
Recall
{test_metrics[‘test_recall’]:.4f}
F1
{test_metrics[‘test_f1’]:.4f}
ROC AUC
{test_metrics[‘test_roc_auc’]:.4f}
Leakage Notes

StandardScaler is inside Pipeline, so it is fit only on each training fold.
The holdout test set is not used during GridSearchCV.
The chosen model is evaluated on the test set only after model selection.

Interpretation
Write your own explanation here:

Is the best CV model also clearly better than the others, or only slightly better?
Does the chosen metric match the business or scientific goal?
Are the best parameters reasonable, or do they suggest overfitting?
What should be checked with a larger dataset or repeated CV?
"""
return report

def main():
X, y, target_names = load_data()
X_train, X_test, y_train, y_test = train_test_holdout(X, y)
plain results = run_model_selection(X_train, y_train)
best_model = results[0][“best_estimator”]
test_metrics = evaluate_classifier(best_model, X_test, y_test)
plain report = make_report(results, test_metrics, target_names)
REPORT_PATH.parent.mkdir(parents=True, exist_ok=True)
REPORT_PATH.write_text(report, encoding=“utf-8”)
plain display_rows = [
{
“model”: row[“model”],
“mean_cv_f1”: round(row[“best_score”], 4),
“best_params”: row[“best_params”],
}
for row in results
]
print(pd.DataFrame(display_rows).to_string(index=False))
print()
print(“Best model:”, results[0][“model”])
print(“Test metrics:”, {key: round(value, 4) for key, value in test_metrics.items()})
print(“Report written to”, REPORT_PATH)
if name == “main”:
main()

source .venv/bin/activate.fish
python src/model_selection.py

              model  mean_cv_f1                                         best_params
      random_forest      0.9720 {'model__max_depth': None, 'model__min_samples_leaf': 1, 'model__n_estimators': 200}
logistic_regression      0.9680                     {'model__C': 1.0, 'model__class_weight': None}
                knn      0.9662                 {'model__n_neighbors': 9, 'model__weights': 'uniform'}
      decision_tree      0.9371                     {'model__max_depth': 4, 'model__min_samples_leaf': 3}
Best model: random_forest
Test metrics: {‘test_accuracy’: 0.9649, ‘test_precision’: 0.9589, ‘test_recall’: 0.9859, ‘test_f1’: 0.9722, ‘test_roc_auc’: 0.9951}
Report written to reports/week06_cv_report.md

╭─ model selection ─────────────────────╮
│ model                 valid_score      │
├────────────────────────────────────────┤
│ logistic_regression   0.9680           │
│ random_forest         0.9720           │
│ knn                   0.9662           │
│ decision_tree         0.9371           │
╰────────────────────────────────────────╯

best_index = search.best_index_
best_std = search.cv_results_["std_test_score"][best_index]

{
    "model": model_name,
    "best_score": search.best_score_,
    "best_std": best_std,
    "best_params": search.best_params_,
    "best_estimator": search.best_estimator_,
}

| Model | Mean CV F1 | Std | Best Params |

from sklearn.model_selection import KFold
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
cv = KFold(n_splits=5, shuffle=True, random_state=42)

scoring="neg_mean_absolute_error"

mean_mae = -search.best_score_

city, gender, industry, education

from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder, StandardScaler
numeric_features = [“age”, “income”, “score”]
categorical_features = [“city”, “gender”]
preprocess = ColumnTransformer(
transformers=[
(“num”, StandardScaler(), numeric_features),
(“cat”, OneHotEncoder(handle_unknown=“ignore”), categorical_features),
]
)

pipeline = Pipeline(
    steps=[
        ("preprocess", preprocess),
        ("model", LogisticRegression(max_iter=3000)),
    ]
)

code reports/week06_cv_report.md

## Model Choice
为什么选择这个模型？它是否明显优于其他模型？
Metric Choice
为什么用 F1 / AUC / MAE / R²？这个指标和问题目标是否一致？
Leakage Prevention
哪些预处理被放进 Pipeline？test 集有没有参与调参？
Limitations
数据量、特征、划分方式、参数网格有哪些限制？

git status --short
git add pyproject.toml uv.lock README.md src reports
git commit -m "Add week06 cross validation model selection"

.venv/
__pycache__/
.ipynb_checkpoints/

source .venv/bin/activate.fish
python src/model_selection.py
test -f reports/week06_cv_report.md; and echo "report exists"

scaler.fit_transform(X)
然后再 cross_validate

StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

RANDOM_STATE = 42