import matplotlib.pyplot as plt
import numpy as np
from io import StringIO
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier

# Pure noise: no configuration can truly beat 50%. We try K random tree
# configurations, keep the one with the best VALIDATION score, and also look
# at what that same winner scores on an untouched TEST set.
rng = np.random.default_rng(0)
K, REPEATS = 100, 30
best_val, its_test = np.zeros(K), np.zeros(K)
for r in range(REPEATS):
    X = rng.normal(size=(400, 20))
    y = rng.integers(0, 2, 400)
    X_tmp, X_te, y_tmp, y_te = train_test_split(X, y, test_size=0.25, random_state=r)
    X_tr, X_va, y_tr, y_va = train_test_split(X_tmp, y_tmp, test_size=1/3, random_state=r)
    vals, tests = [], []
    for k in range(K):
        model = DecisionTreeClassifier(max_depth=int(rng.integers(1, 8)),
                                       max_features=int(rng.integers(1, 21)),
                                       random_state=k).fit(X_tr, y_tr)
        vals.append(model.score(X_va, y_va))
        tests.append(model.score(X_te, y_te))
        winner = int(np.argmax(vals))
        best_val[k] += vals[winner]
        its_test[k] += tests[winner]
best_val, its_test = best_val / REPEATS * 100, its_test / REPEATS * 100

tried = np.arange(1, K + 1)
fig, ax = plt.subplots(figsize=(6.5, 3.2))
ax.plot(tried, best_val, color='#f0883e', lw=2, label='validation score of the winner')
ax.plot(tried, its_test, color='#58a6ff', lw=2, label='test score of that same winner')
ax.axhline(50, color='gray', ls='--', lw=1)
ax.text(K, 48.3, 'chance: the truth on pure noise', ha='right', fontsize=8, color='gray')
ax.annotate(f'{best_val[-1]:.1f}%', xy=(K, best_val[-1]), xytext=(K - 14, best_val[-1] + 2),
            fontsize=8, arrowprops=dict(arrowstyle='->', color='gray'))
ax.set_xscale('log')
ax.set_xticks([1, 2, 5, 10, 20, 50, 100], ['1', '2', '5', '10', '20', '50', '100'])
ax.set_xlabel('number of configurations compared on the validation set')
ax.set_ylabel('accuracy (%)')
ax.set_ylim(45, 66)
ax.legend(fontsize=8, loc='upper left')
ax.set_title('Choosing on a set makes that set optimistic (pure-noise data)', fontsize=10)

plt.tight_layout()

buffer = StringIO()
plt.savefig(buffer, format="svg", transparent=True)
print(buffer.getvalue())

plt.close()