import matplotlib.pyplot as plt
import numpy as np
from io import StringIO
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold, cross_val_score, train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

# Same data, same model; the only thing that changes is the random seed of the split
X, y = load_breast_cancer(return_X_y=True)
pipe = make_pipeline(StandardScaler(), LogisticRegression(max_iter=1000))

holdout = []
for seed in range(300):
    X_tr, X_va, y_tr, y_va = train_test_split(X, y, test_size=0.2, random_state=seed, stratify=y)
    holdout.append(pipe.fit(X_tr, y_tr).score(X_va, y_va))

cv_mean = []
for seed in range(100):
    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed)
    cv_mean.append(cross_val_score(pipe, X, y, cv=cv).mean())

holdout, cv_mean = np.array(holdout) * 100, np.array(cv_mean) * 100
step = 100 / 114                      # one validation row = 0.88 points of accuracy
bins = np.arange(92.5, 101.5, step) - step / 2 + (100 % step)

fig, (top, bottom) = plt.subplots(2, 1, figsize=(6.5, 3.8), sharex=True)
top.hist(holdout, bins=bins, color='#f0883e', rwidth=0.85)
top.set_title(f'one 80/20 split, 300 different seeds: from {holdout.min():.1f}% to {holdout.max():.1f}%',
              fontsize=9, loc='left')
bottom.hist(cv_mean, bins=np.arange(92.5, 100.6, 0.2), color='#58a6ff', rwidth=0.85)
bottom.set_title(f'mean of 5-fold CV, 100 different seeds: from {cv_mean.min():.1f}% to {cv_mean.max():.1f}%',
                 fontsize=9, loc='left')
for ax in (top, bottom):
    ax.set_ylabel('seeds', fontsize=8)
    ax.tick_params(labelsize=8)
bottom.set_xlabel('estimated accuracy (%)')
bottom.set_xlim(92.5, 100.7)
fig.suptitle('Same data, same model: only the random seed of the split changes', fontsize=10)

plt.tight_layout()

buffer = StringIO()
plt.savefig(buffer, format="svg", transparent=True)
print(buffer.getvalue())

plt.close()