import sklearn.datasets
import sklearn.metrics

import autosklearn.classification

X, y = sklearn.datasets.load_breast_cancer(return_X_y=True)
import numpy as np

rand = np.random.RandomState(2)
X = rand.random((100, 50))
y = rand.randint(0, 2, (100,))
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
    X, y, random_state=1
)


def callback(
    smbo,
    run_info,
    result,
    time_left,
):
    if int(result.additional_info["num_run"]) > 0:
        return False


automl = autosklearn.classification.AutoSklearnClassifier(
    time_left_for_this_task=120,
    per_run_time_limit=30,
    get_trials_callback=callback,
    include={"classifier": ["bernoulli_nb"]},
    delete_tmp_folder_after_terminate=False
)
automl.fit(X_train, y_train, dataset_name="breast_cancer")

for ens in automl.get_pareto_set():
    ens.predict(X_test)

Traceback (most recent call last):
  File "test.py", line 41, in <module>
    ens.predict(X_test)
  File "/home/skantify/code/asklearn/dev/.venv/lib/python3.8/site-packages/sklearn/ensemble/_voting.py", line 309, in predict
    maj = np.argmax(self.predict_proba(X), axis=1)
  File "/home/skantify/code/asklearn/dev/.venv/lib/python3.8/site-packages/sklearn/ensemble/_voting.py", line 330, in _predict_proba
    weights=self._weights_not_none)
  File "/home/skantify/code/asklearn/dev/.venv/lib/python3.8/site-packages/sklearn/ensemble/_voting.py", line 56, in _weights_not_none
    return [w for est, w in zip(self.estimators, self.weights)
  File "/home/skantify/code/asklearn/dev/.venv/lib/python3.8/site-packages/sklearn/ensemble/_voting.py", line 57, in <listcomp>
    if est[1] != 'drop']
TypeError: 'MyDummyClassifier' object is not subscriptable

@case(tags=["classifier", "fitted"])
def case_classifier_fitted_only_dummy(
    make_cache: Callable[[str], Cache],
    make_backend: Callable[..., Backend],
    make_automl_classifier: Callable[..., AutoMLClassifier],
) -> AutoMLClassifier:
    """Case of a fitted classifier but only dummy was found"""
    key = "case_classifier_fitted_only_dummy"

    # This locks the cache for this item while we check, required for pytest-xdist

    with make_cache(key) as cache:
        if "model" not in cache:
            model = make_automl_classifier(
                temporary_directory=cache.path("backend"),
                delete_tmp_folder_after_terminate=False,
                include={"classifier": ["bernoulli_nb"]},  # Just a meh model
                get_trials_callback=stop_at_first,
            )
            rand = np.random.RandomState(2)
            _X = rand.random((100, 50))
            _y = rand.randint(0, 2, (100,))
            X, Xt, y, yt = sklearn.model_selection.train_test_split(
                _X, _y, random_state=1  # Required to ensure dummy is best
            )
            model.fit(X, y, dataset_name="random")

            # We now validate that indeed, the only model is the Dummy
            members = list(model.models_.values())
            if len(members) != 1 and not isinstance(members[0], MyDummyClassifier):
                raise ValueError("Should only have one model, dummy\n", members)

            cache.save(model, "model")

    model = cache.load("model")
    model._backend = copy_backend(old=model._backend, new=make_backend())

    return model