Skip to content

Commit 1036020

Browse files
author
Github Actions
committed
Ravin Kohli: [ADD] fit pipeline honoring API constraints with tests (#348)
1 parent a19b931 commit 1036020

File tree

46 files changed

+1975
-560
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+1975
-560
lines changed

‎development/_downloads/3b0b756ccfcac69e6a1673e56f2f543f/example_visualization.ipynb

+1-1
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@
9898
},
9999
"outputs": [],
100100
"source": [
101-
"# We will plot the search incumbent through time.\n\n# Collect the performance of individual machine learning algorithms\n# found by SMAC\nindividual_performances = []\nfor run_key, run_value in estimator.run_history.data.items():\n if run_value.status != StatusType.SUCCESS:\n # Ignore crashed runs\n continue\n individual_performances.append({\n 'Timestamp': pd.Timestamp(\n time.strftime(\n '%Y-%m-%d %H:%M:%S',\n time.localtime(run_value.endtime)\n )\n ),\n 'single_best_optimization_accuracy': accuracy._optimum - run_value.cost,\n 'single_best_test_accuracy': np.nan if run_value.additional_info is None else\n accuracy._optimum - run_value.additional_info['test_loss']['accuracy'],\n })\nindividual_performance_frame = pd.DataFrame(individual_performances)\n\n# Collect the performance of the ensemble through time\n# This ensemble is built from the machine learning algorithms\n# found by SMAC\nensemble_performance_frame = pd.DataFrame(estimator.ensemble_performance_history)\n\n# As we are tracking the incumbent, we are interested in the cummax() performance\nensemble_performance_frame['ensemble_optimization_accuracy'] = ensemble_performance_frame[\n 'train_accuracy'\n].cummax()\nensemble_performance_frame['ensemble_test_accuracy'] = ensemble_performance_frame[\n 'test_accuracy'\n].cummax()\nensemble_performance_frame.drop(columns=['test_accuracy', 'train_accuracy'], inplace=True)\nindividual_performance_frame['single_best_optimization_accuracy'] = individual_performance_frame[\n 'single_best_optimization_accuracy'\n].cummax()\nindividual_performance_frame['single_best_test_accuracy'] = individual_performance_frame[\n 'single_best_test_accuracy'\n].cummax()\n\npd.merge(\n ensemble_performance_frame,\n individual_performance_frame,\n on=\"Timestamp\", how='outer'\n).sort_values('Timestamp').fillna(method='ffill').plot(\n x='Timestamp',\n kind='line',\n legend=True,\n title='Auto-PyTorch accuracy over time',\n grid=True,\n)\nplt.show()\n\n# We then can understand the importance of each input feature using\n# a permutation importance analysis. This is done as a proof of concept, to\n# showcase that we can leverage of scikit-learn API.\nresult = permutation_importance(estimator, X_train, y_train, n_repeats=5,\n scoring='accuracy',\n random_state=seed)\nsorted_idx = result.importances_mean.argsort()\n\nfig, ax = plt.subplots()\nax.boxplot(result.importances[sorted_idx].T,\n vert=False, labels=X_test.columns[sorted_idx])\nax.set_title(\"Permutation Importances (Train set)\")\nfig.tight_layout()\nplt.show()"
101+
"# We will plot the search incumbent through time.\n\n# Collect the performance of individual machine learning algorithms\n# found by SMAC\nindividual_performances = []\nfor run_key, run_value in estimator.run_history.data.items():\n if run_value.status != StatusType.SUCCESS:\n # Ignore crashed runs\n continue\n individual_performances.append({\n 'Timestamp': pd.Timestamp(\n time.strftime(\n '%Y-%m-%d %H:%M:%S',\n time.localtime(run_value.endtime)\n )\n ),\n 'single_best_optimization_accuracy': accuracy._optimum - run_value.cost,\n 'single_best_test_accuracy': np.nan if run_value.additional_info is None else\n accuracy._optimum - run_value.additional_info['test_loss']['accuracy'],\n })\nindividual_performance_frame = pd.DataFrame(individual_performances)\n\n# Collect the performance of the ensemble through time\n# This ensemble is built from the machine learning algorithms\n# found by SMAC\nensemble_performance_frame = pd.DataFrame(estimator.ensemble_performance_history)\n\n# As we are tracking the incumbent, we are interested in the cummax() performance\nensemble_performance_frame['ensemble_optimization_accuracy'] = ensemble_performance_frame[\n 'train_accuracy'\n].cummax()\nensemble_performance_frame['ensemble_test_accuracy'] = ensemble_performance_frame[\n 'test_accuracy'\n].cummax()\nensemble_performance_frame.drop(columns=['test_accuracy', 'train_accuracy'], inplace=True)\nindividual_performance_frame['single_best_optimization_accuracy'] = individual_performance_frame[\n 'single_best_optimization_accuracy'\n].cummax()\nindividual_performance_frame['single_best_test_accuracy'] = individual_performance_frame[\n 'single_best_test_accuracy'\n].cummax()\n\npd.merge(\n ensemble_performance_frame,\n individual_performance_frame,\n on=\"Timestamp\", how='outer'\n).sort_values('Timestamp').fillna(method='ffill').plot(\n x='Timestamp',\n kind='line',\n legend=True,\n title='Auto-PyTorch accuracy over time',\n grid=True,\n)\nplt.show()"
102102
]
103103
}
104104
],

‎development/_downloads/83b132442a977189d27fd09560b2bd34/example_single_configuration.ipynb

+126
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"metadata": {
7+
"collapsed": false
8+
},
9+
"outputs": [],
10+
"source": [
11+
"%matplotlib inline"
12+
]
13+
},
14+
{
15+
"cell_type": "markdown",
16+
"metadata": {},
17+
"source": [
18+
"\n# Fit a single configuration\n*Auto-PyTorch* searches for the best combination of machine learning algorithms\nand their hyper-parameter configuration for a given task.\nThis example shows how one can fit one of these pipelines, both, with a user defined\nconfiguration, and a randomly sampled one form the configuration space.\nThe pipelines that Auto-PyTorch fits are compatible with Scikit-Learn API. You can\nget further documentation about Scikit-Learn models here: <https://scikit-learn.org/stable/getting_started.html`>_\n"
19+
]
20+
},
21+
{
22+
"cell_type": "code",
23+
"execution_count": null,
24+
"metadata": {
25+
"collapsed": false
26+
},
27+
"outputs": [],
28+
"source": [
29+
"import os\nimport tempfile as tmp\nimport warnings\n\nos.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir()\nos.environ['OMP_NUM_THREADS'] = '1'\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\nos.environ['MKL_NUM_THREADS'] = '1'\n\nwarnings.simplefilter(action='ignore', category=UserWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport sklearn.datasets\nimport sklearn.metrics\n\nfrom autoPyTorch.api.tabular_classification import TabularClassificationTask\nfrom autoPyTorch.datasets.resampling_strategy import HoldoutValTypes"
30+
]
31+
},
32+
{
33+
"cell_type": "markdown",
34+
"metadata": {},
35+
"source": [
36+
"## Data Loading\n\n"
37+
]
38+
},
39+
{
40+
"cell_type": "code",
41+
"execution_count": null,
42+
"metadata": {
43+
"collapsed": false
44+
},
45+
"outputs": [],
46+
"source": [
47+
"X, y = sklearn.datasets.fetch_openml(data_id=3, return_X_y=True, as_frame=True)\nX_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(\n X, y, test_size=0.5, random_state=3\n)"
48+
]
49+
},
50+
{
51+
"cell_type": "markdown",
52+
"metadata": {},
53+
"source": [
54+
"## Define an estimator\n\n"
55+
]
56+
},
57+
{
58+
"cell_type": "code",
59+
"execution_count": null,
60+
"metadata": {
61+
"collapsed": false
62+
},
63+
"outputs": [],
64+
"source": [
65+
"estimator = TabularClassificationTask(\n resampling_strategy=HoldoutValTypes.holdout_validation,\n resampling_strategy_args={'val_share': 0.5},\n)"
66+
]
67+
},
68+
{
69+
"cell_type": "markdown",
70+
"metadata": {},
71+
"source": [
72+
"## Get a configuration of the pipeline for current dataset\n\n"
73+
]
74+
},
75+
{
76+
"cell_type": "code",
77+
"execution_count": null,
78+
"metadata": {
79+
"collapsed": false
80+
},
81+
"outputs": [],
82+
"source": [
83+
"dataset = estimator.get_dataset(X_train=X_train,\n y_train=y_train,\n X_test=X_test,\n y_test=y_test,\n dataset_name='kr-vs-kp')\nconfiguration = estimator.get_search_space(dataset).get_default_configuration()\n\nprint(\"Passed Configuration:\", configuration)"
84+
]
85+
},
86+
{
87+
"cell_type": "markdown",
88+
"metadata": {},
89+
"source": [
90+
"## Fit the configuration\n\n"
91+
]
92+
},
93+
{
94+
"cell_type": "code",
95+
"execution_count": null,
96+
"metadata": {
97+
"collapsed": false
98+
},
99+
"outputs": [],
100+
"source": [
101+
"pipeline, run_info, run_value, dataset = estimator.fit_pipeline(dataset=dataset,\n configuration=configuration,\n budget_type='epochs',\n budget=10,\n run_time_limit_secs=100\n )\n\n# The fit_pipeline command also returns a named tuple with the pipeline constraints\nprint(run_info)\n\n# The fit_pipeline command also returns a named tuple with train/test performance\nprint(run_value)\n\n# This object complies with Scikit-Learn Pipeline API.\n# https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html\nprint(pipeline.named_steps)"
102+
]
103+
}
104+
],
105+
"metadata": {
106+
"kernelspec": {
107+
"display_name": "Python 3",
108+
"language": "python",
109+
"name": "python3"
110+
},
111+
"language_info": {
112+
"codemirror_mode": {
113+
"name": "ipython",
114+
"version": 3
115+
},
116+
"file_extension": ".py",
117+
"mimetype": "text/x-python",
118+
"name": "python",
119+
"nbconvert_exporter": "python",
120+
"pygments_lexer": "ipython3",
121+
"version": "3.8.12"
122+
}
123+
},
124+
"nbformat": 4,
125+
"nbformat_minor": 0
126+
}

‎development/_downloads/a4083e360cf01f594602cbaf737091b0/example_visualization.py

-15
Original file line numberDiff line numberDiff line change
@@ -149,18 +149,3 @@
149149
grid=True,
150150
)
151151
plt.show()
152-
153-
# We then can understand the importance of each input feature using
154-
# a permutation importance analysis. This is done as a proof of concept, to
155-
# showcase that we can leverage of scikit-learn API.
156-
result = permutation_importance(estimator, X_train, y_train, n_repeats=5,
157-
scoring='accuracy',
158-
random_state=seed)
159-
sorted_idx = result.importances_mean.argsort()
160-
161-
fig, ax = plt.subplots()
162-
ax.boxplot(result.importances[sorted_idx].T,
163-
vert=False, labels=X_test.columns[sorted_idx])
164-
ax.set_title("Permutation Importances (Train set)")
165-
fig.tight_layout()
166-
plt.show()

‎development/_downloads/b71f91b170d4a46b842cd9568511ced2/example_single_configuration.py

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# -*- encoding: utf-8 -*-
2+
"""
3+
==========================
4+
Fit a single configuration
5+
==========================
6+
*Auto-PyTorch* searches for the best combination of machine learning algorithms
7+
and their hyper-parameter configuration for a given task.
8+
This example shows how one can fit one of these pipelines, both, with a user defined
9+
configuration, and a randomly sampled one form the configuration space.
10+
The pipelines that Auto-PyTorch fits are compatible with Scikit-Learn API. You can
11+
get further documentation about Scikit-Learn models here: <https://scikit-learn.org/stable/getting_started.html`>_
12+
"""
13+
import os
14+
import tempfile as tmp
15+
import warnings
16+
17+
os.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir()
18+
os.environ['OMP_NUM_THREADS'] = '1'
19+
os.environ['OPENBLAS_NUM_THREADS'] = '1'
20+
os.environ['MKL_NUM_THREADS'] = '1'
21+
22+
warnings.simplefilter(action='ignore', category=UserWarning)
23+
warnings.simplefilter(action='ignore', category=FutureWarning)
24+
25+
import sklearn.datasets
26+
import sklearn.metrics
27+
28+
from autoPyTorch.api.tabular_classification import TabularClassificationTask
29+
from autoPyTorch.datasets.resampling_strategy import HoldoutValTypes
30+
31+
32+
############################################################################
33+
# Data Loading
34+
# ============
35+
36+
X, y = sklearn.datasets.fetch_openml(data_id=3, return_X_y=True, as_frame=True)
37+
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
38+
X, y, test_size=0.5, random_state=3
39+
)
40+
41+
############################################################################
42+
# Define an estimator
43+
# ===================
44+
45+
estimator = TabularClassificationTask(
46+
resampling_strategy=HoldoutValTypes.holdout_validation,
47+
resampling_strategy_args={'val_share': 0.5},
48+
)
49+
50+
############################################################################
51+
# Get a configuration of the pipeline for current dataset
52+
# ===============================================================
53+
54+
dataset = estimator.get_dataset(X_train=X_train,
55+
y_train=y_train,
56+
X_test=X_test,
57+
y_test=y_test,
58+
dataset_name='kr-vs-kp')
59+
configuration = estimator.get_search_space(dataset).get_default_configuration()
60+
61+
print("Passed Configuration:", configuration)
62+
###########################################################################
63+
# Fit the configuration
64+
# =====================
65+
66+
pipeline, run_info, run_value, dataset = estimator.fit_pipeline(dataset=dataset,
67+
configuration=configuration,
68+
budget_type='epochs',
69+
budget=10,
70+
run_time_limit_secs=100
71+
)
72+
73+
# The fit_pipeline command also returns a named tuple with the pipeline constraints
74+
print(run_info)
75+
76+
# The fit_pipeline command also returns a named tuple with train/test performance
77+
print(run_value)
78+
79+
# This object complies with Scikit-Learn Pipeline API.
80+
# https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html
81+
print(pipeline.named_steps)

‎development/_downloads/bc82bea3a5dd7bdba60b65220891d9e5/examples_python.zip

2.75 KB
Binary file not shown.

‎development/_downloads/fb625db3c50d423b1b7881136ffdeec8/examples_jupyter.zip

4.38 KB
Binary file not shown.

‎development/_images/sphx_glr_example_plot_over_time_001.png

-395 Bytes
Loading

‎development/_images/sphx_glr_example_plot_over_time_thumb.png

-1.21 KB
Loading

‎development/_images/sphx_glr_example_single_configuration_thumb.png

26.2 KB
Loading

‎development/_images/sphx_glr_example_visualization_001.png

3.11 KB
Loading

‎development/_images/sphx_glr_example_visualization_002.png

-17.7 KB
Binary file not shown.

‎development/_images/sphx_glr_example_visualization_thumb.png

2.32 KB
Loading

0 commit comments

Comments
 (0)