add autodl

AutoDL-Projects/notebooks/NATS-Bench/BayesOpt.ipynb

@@ -0,0 +1,118 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "german-madonna",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Implementation for \"A Tutorial on Bayesian Optimization\"\n",
+    "import numpy as np\n",
+    "\n",
+    "def get_data():\n",
+    "    return np.random.random(2) * 10\n",
+    "\n",
+    "def f(x):\n",
+    "    return float(np.power((x[0] * 3 - x[1]), 3) - np.exp(x[1]) + np.power(x[0], 2))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "broke-citizenship",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Kernels typically have the property that points closer in the input space are more strongly correlated\n",
+    "# i.e., if |x1 - x2| < |x1 - x3|, then sigma(x1, x2) > sigma(x1, x3).\n",
+    "# the commonly used and simple kernel is the power exponential or Gaussian kernel:\n",
+    "def sigma0(x1, x2, alpha0=1, alpha=[1,1]):\n",
+    "    \"\"\"alpha could be a vector\"\"\"\n",
+    "    power = np.array(alpha, dtype=np.float32) * np.power(np.array(x1)-np.array(x2), 2)\n",
+    "    return alpha0 * np.exp( -np.sum(power) )\n",
+    "\n",
+    "# the most common choice for the mean function is a constant value\n",
+    "def mu0(x, mu):\n",
+    "    return mu"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "aerial-carnival",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "K = 5\n",
+    "X = np.array([get_data() for i in range(K)])\n",
+    "mu = np.mean(X, axis=0)\n",
+    "mu0_over_K = [mu0(x, mu) for x in X]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "polished-discussion",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sigma0_over_KK = []\n",
+    "for i in range(K):\n",
+    "    sigma0_over_KK.append(np.array([sigma0(X[i], X[j]) for j in range(K)]))\n",
+    "sigma0_over_KK = np.array(sigma0_over_KK)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "comic-jesus",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(20, 20)\n",
+      "1.1038803861344952e-06\n",
+      "1.1038803861344952e-06\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(sigma0_over_KK.shape)\n",
+    "print(sigma0_over_KK[1][2])\n",
+    "print(sigma0_over_KK[2][1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "statistical-wrist",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

AutoDL-Projects/notebooks/NATS-Bench/find-largest.ipynb

@@ -0,0 +1,88 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2021-03-27 06:46:38] Try to use the default NATS-Bench (topology) path from fast_mode=True and path=None.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from nats_bench import create\n",
+    "from pprint import pprint\n",
+    "# Create the API for tologoy search space\n",
+    "api = create(None, 'tss', fast_mode=True, verbose=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'test-accuracy': 22.39999992879232,\n",
+      " 'test-all-time': 7.7054752962929856,\n",
+      " 'test-loss': 3.1626377182006835,\n",
+      " 'test-per-time': 0.6421229413577488,\n",
+      " 'train-accuracy': 21.68885959195242,\n",
+      " 'train-all-time': 1260.0195466594694,\n",
+      " 'train-loss': 3.1863493608815463,\n",
+      " 'train-per-time': 105.00162888828912,\n",
+      " 'valid-accuracy': 23.266666631062826,\n",
+      " 'valid-all-time': 7.7054752962929856,\n",
+      " 'valid-loss': 3.1219845104217527,\n",
+      " 'valid-per-time': 0.6421229413577488,\n",
+      " 'valtest-accuracy': 22.833333323160808,\n",
+      " 'valtest-all-time': 15.410950592585971,\n",
+      " 'valtest-loss': 3.142311067581177,\n",
+      " 'valtest-per-time': 1.2842458827154977}\n"
+     ]
+    }
+   ],
+   "source": [
+    "largest_candidate_tss = '|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|'\n",
+    "\n",
+    "arch_index = api.query_index_by_arch(largest_candidate_tss)\n",
+    "info = api.get_more_info(arch_index, 'ImageNet16-120', hp='12', is_random=False)\n",
+    "pprint(info)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

AutoDL-Projects/notebooks/NATS-Bench/issue-96.ipynb

@@ -0,0 +1,91 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2021-03-01 12:28:12] Try to use the default NATS-Bench (topology) path from fast_mode=True and path=None.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from nats_bench import create\n",
+    "import numpy as np\n",
+    "\n",
+    "def get_correlation(A, B):\n",
+    "    return float(np.corrcoef(A, B)[0,1])\n",
+    "\n",
+    "# Create the API for tologoy search space\n",
+    "api = create(None, 'tss', fast_mode=True, verbose=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "There are 15625 architectures on the topology search space\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('There are {:} architectures on the topology search space'.format(len(api)))\n",
+    "accuracies_12, accuracies_200 = [], []\n",
+    "for i, arch in enumerate(api):\n",
+    "    info_a = api.get_more_info(i, dataset='cifar10-valid', hp='12', is_random=False)\n",
+    "    accuracies_12.append(info_a['valid-accuracy'])\n",
+    "\n",
+    "    info_b = api.get_more_info(i, dataset='cifar10-valid', hp='200', is_random=False)\n",
+    "    accuracies_200.append(info_b['test-accuracy'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[CIFAR-10] The correlation between 12-epoch validation accuracy and 200-epoch test accuracy is: 91.18%\n"
+     ]
+    }
+   ],
+   "source": [
+    "correlation = get_correlation(accuracies_12, accuracies_200)\n",
+    "print('[CIFAR-10] The correlation between 12-epoch validation accuracy and 200-epoch test accuracy is: {:.2f}%'.format(correlation * 100))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

AutoDL-Projects/notebooks/NATS-Bench/issue-97.ipynb

@@ -0,0 +1,86 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2021-03-09 08:44:19] Try to use the default NATS-Bench (size) path from fast_mode=True and path=None.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from nats_bench import create\n",
+    "import numpy as np\n",
+    "\n",
+    "# Create the API for size search space\n",
+    "api = create(None, 'sss', fast_mode=True, verbose=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "There are 32768 architectures on the size search space\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('There are {:} architectures on the size search space'.format(len(api)))\n",
+    "\n",
+    "c2acc = dict()\n",
+    "for index in range(len(api)):\n",
+    "    info = api.get_more_info(index, 'cifar10', hp='90')\n",
+    "    config = api.get_net_config(index, 'cifar10')\n",
+    "    c2acc[config['channels']] = info['test-accuracy']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "91.08546417236329\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(np.mean(list(c2acc.values())))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}