plot_nasbench201.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "liberal-hawaii",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "hairy-machine",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_plot_data(results, grouby='acc_test_best'):\n",
    "  d = {}\n",
    "  d['num_sample'] = []\n",
    "  d['acc_valid_best'] = []\n",
    "  d['acc_test_best'] = []\n",
    "  for result in results:\n",
    "    num_sample = result['num_unique_sample']\n",
    "    acc_valid = result['acc_valid']\n",
    "    acc_test = result['acc_test']\n",
    "    index_unique_sample = result['index_unique_sample']\n",
    "    index_pred_top = result['index_pred_top']\n",
    "    for i in range(len(index_unique_sample)):\n",
    "      d['num_sample'].append(num_sample[i])\n",
    "      d['acc_valid_best'].append(max(np.max(acc_valid[index_unique_sample[i]]),acc_valid[index_pred_top[i][0]]))\n",
    "      d['acc_test_best'].append(max(np.max(acc_test[index_unique_sample[i]]),acc_test[index_pred_top[i][0]]))\n",
    "\n",
    "  run = len(results)\n",
    "  df = pd.DataFrame(data=d)\n",
    "  grouped = df[grouby].groupby(df['num_sample'])\n",
    "  est = grouped.agg(\"mean\")\n",
    "  sd = grouped.std()\n",
    "  sd = sd.fillna(value=0)\n",
    "  cis = pd.DataFrame(np.c_[est - sd, est + sd], index=est.index, columns=[\"low\", \"high\"]).stack()\n",
    "  if cis.notnull().any():\n",
    "      cis = cis.unstack().reindex(est.index)\n",
    "  else:\n",
    "      cis = None\n",
    "  x, y = np.asarray(est.index), np.asarray(est)\n",
    "  low, high = np.asarray(cis[\"low\"]), np.asarray(cis[\"high\"])\n",
    "  std = np.asarray(sd)\n",
    "  plot_data={\"x\":x,\"y\":y,'var':std}\n",
    "  return plot_data\n",
    "\n",
    "def draw_baseline(ax, data_dict):\n",
    "  for key, value in data_dict.items():\n",
    "    p = ax.plot(value[\"x\"], value[\"y\"], label=key)\n",
    "    if \"var\" in value:\n",
    "      ax.fill_between(value[\"x\"], value[\"y\"] - value[\"var\"], value[\"y\"] + value[\"var\"], alpha=0.1, color=p[0].get_color(), edgecolor=None)\n",
    "  return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "blank-qualification",
   "metadata": {},
   "outputs": [],
   "source": [
    "# NASBench201 CIFAR10 Subset\n",
    "base_dir = 'OUTPUT/nasbench201/cifar100' # modify base_dir to log path\n",
    "exp_dict = {'WeakNAS':\n",
    "            'init_10_sample_10/top_start_100_end_100/MLP/onehot_size_1000_1000_1000_1000_iter_100/acq_uniform/deter_False',\n",
    "            'WeakNAS (EI Variant)':\n",
    "            'init_10_sample_10/top_start_100_end_100/MLP/onehot_size_1000_1000_1000_1000_iter_100/acq_ei/deter_False',}\n",
    "for key, value in exp_dict.items():\n",
    "  exp_dict[key] = os.path.join(base_dir, value)\n",
    "  \n",
    "data_dict = {}\n",
    "for key, exp_dir in exp_dict.items():\n",
    "  files = [os.path.join(exp_dir, f) for f in os.listdir(exp_dir)]\n",
    "  results = []\n",
    "  for file in files:\n",
    "    with open(file, 'rb') as f:\n",
    "      results.append(pickle.load(f))\n",
    "  data_dict[key] = get_plot_data(results, grouby='acc_valid_best')\n",
    "\n",
    "fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(7, 5), facecolor='white')\n",
    "ax.set_facecolor(color=(1.0, 1.0, 1.0, 0.0))\n",
    "ax = draw_baseline(ax, data_dict)\n",
    "ax.set_ylabel('Best Validation Accuracy')\n",
    "ax.set_xlabel('Number of Queries')\n",
    "leg = ax.legend(loc='lower right')\n",
    "\n",
    "final_archs = {}\n",
    "data_dict = {}\n",
    "for key, exp_dir in exp_dict.items():\n",
    "  files = [os.path.join(exp_dir, f) for f in os.listdir(exp_dir)]\n",
    "  results = []\n",
    "  for file in files:\n",
    "    with open(file, 'rb') as f:\n",
    "      results.append(pickle.load(f))\n",
    "  data = get_plot_data(results, grouby='acc_test_best')\n",
    "  data_dict[key] = data\n",
    "  \n",
    "fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(7, 5), facecolor='white')\n",
    "ax.set_facecolor(color=(1.0, 1.0, 1.0, 0.0))\n",
    "ax = draw_baseline(ax, data_dict)\n",
    "ax.set_ylabel('Best Test Accuracy')\n",
    "ax.set_xlabel('Number of Queries')\n",
    "leg = ax.legend(loc='lower right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "hazardous-makeup",
   "metadata": {},
   "outputs": [],
   "source": [
    "# NASBench201 CIFAR100 Subset\n",
    "base_dir = 'OUTPUT/nasbench201/cifar100' # modify base_dir to log path\n",
    "exp_dict = {'WeakNAS':\n",
    "            'init_10_sample_10/top_start_100_end_100/MLP/onehot_size_1000_1000_1000_1000_iter_100/acq_uniform/deter_False',\n",
    "            'WeakNAS (EI Variant)':\n",
    "            'init_10_sample_10/top_start_100_end_100/MLP/onehot_size_1000_1000_1000_1000_iter_100/acq_ei/deter_False',}\n",
    "for key, value in exp_dict.items():\n",
    "  exp_dict[key] = os.path.join(base_dir, value)\n",
    "  \n",
    "data_dict = {}\n",
    "for key, exp_dir in exp_dict.items():\n",
    "  files = [os.path.join(exp_dir, f) for f in os.listdir(exp_dir)]\n",
    "  results = []\n",
    "  for file in files:\n",
    "    with open(file, 'rb') as f:\n",
    "      results.append(pickle.load(f))\n",
    "  data_dict[key] = get_plot_data(results, grouby='acc_valid_best')\n",
    "\n",
    "fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(7, 5), facecolor='white')\n",
    "ax.set_facecolor(color=(1.0, 1.0, 1.0, 0.0))\n",
    "ax = draw_baseline(ax, data_dict)\n",
    "ax.set_ylabel('Best Validation Accuracy')\n",
    "ax.set_xlabel('Number of Queries')\n",
    "leg = ax.legend(loc='lower right')\n",
    "\n",
    "final_archs = {}\n",
    "data_dict = {}\n",
    "for key, exp_dir in exp_dict.items():\n",
    "  files = [os.path.join(exp_dir, f) for f in os.listdir(exp_dir)]\n",
    "  results = []\n",
    "  for file in files:\n",
    "    with open(file, 'rb') as f:\n",
    "      results.append(pickle.load(f))\n",
    "  data = get_plot_data(results, grouby='acc_test_best')\n",
    "  data_dict[key] = data\n",
    "  \n",
    "fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(7, 5), facecolor='white')\n",
    "ax.set_facecolor(color=(1.0, 1.0, 1.0, 0.0))\n",
    "ax = draw_baseline(ax, data_dict)\n",
    "ax.set_ylabel('Best Test Accuracy')\n",
    "ax.set_xlabel('Number of Queries')\n",
    "leg = ax.legend(loc='lower right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cellular-sleeve",
   "metadata": {},
   "outputs": [],
   "source": [
    "# NASBench201 ImageNet16-120 Subset\n",
    "base_dir = 'OUTPUT/nasbench201/ImageNet16-120' # modify base_dir to log path\n",
    "exp_dict = {'WeakNAS':\n",
    "            'init_10_sample_10/top_start_100_end_100/MLP/onehot_size_1000_1000_1000_1000_iter_100/acq_uniform/deter_False',\n",
    "            'WeakNAS (EI Variant)':\n",
    "            'init_10_sample_10/top_start_100_end_100/MLP/onehot_size_1000_1000_1000_1000_iter_100/acq_ei/deter_False',}\n",
    "for key, value in exp_dict.items():\n",
    "  exp_dict[key] = os.path.join(base_dir, value)\n",
    "  \n",
    "data_dict = {}\n",
    "for key, exp_dir in exp_dict.items():\n",
    "  files = [os.path.join(exp_dir, f) for f in os.listdir(exp_dir)]\n",
    "  results = []\n",
    "  for file in files:\n",
    "    with open(file, 'rb') as f:\n",
    "      results.append(pickle.load(f))\n",
    "  data_dict[key] = get_plot_data(results, grouby='acc_valid_best')\n",
    "\n",
    "fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(7, 5), facecolor='white')\n",
    "ax.set_facecolor(color=(1.0, 1.0, 1.0, 0.0))\n",
    "ax = draw_baseline(ax, data_dict)\n",
    "ax.set_ylabel('Best Validation Accuracy')\n",
    "ax.set_xlabel('Number of Queries')\n",
    "leg = ax.legend(loc='lower right')\n",
    "\n",
    "final_archs = {}\n",
    "data_dict = {}\n",
    "for key, exp_dir in exp_dict.items():\n",
    "  files = [os.path.join(exp_dir, f) for f in os.listdir(exp_dir)]\n",
    "  results = []\n",
    "  for file in files:\n",
    "    with open(file, 'rb') as f:\n",
    "      results.append(pickle.load(f))\n",
    "  data = get_plot_data(results, grouby='acc_test_best')\n",
    "  data_dict[key] = data\n",
    "  \n",
    "fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(7, 5), facecolor='white')\n",
    "ax.set_facecolor(color=(1.0, 1.0, 1.0, 0.0))\n",
    "ax = draw_baseline(ax, data_dict)\n",
    "ax.set_ylabel('Best Test Accuracy')\n",
    "ax.set_xlabel('Number of Queries')\n",
    "leg = ax.legend(loc='lower right')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}