24. Principal Component Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Principal Component Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$max \\    Var(X_{project}) = \\frac{1}{m} \\sum (X_{project} - \\bar X)^2 = \\frac{1}{m}\\sum(X_i·w)^2$\n",
    "\n",
    "<font size=4>$w$ is the unit transition matrix of the axis  of PCA </font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$f(w) = \\frac{1}{m} \\sum(X_i·w)^2 = \\frac{1}{m} (Xw)'Xw$\n",
    "\n",
    "$\\nabla f(w) =\\frac{2}{m} \\begin{bmatrix} \\partial f/ \\partial w_1 \\\\ \\partial f/ \\partial w_2\\\\ ····· \\\\ \\partial f/ \\partial w_n  \\end{bmatrix} = \\frac{2}{m} \\begin{bmatrix} \\sum_{i = 1}^m (X_i·w)(X_{i1}) \\\\ \\sum_{i = 1}^m (X_i·w)(X_{i2}) \\\\ ····· \\\\ \\sum_{i = 1}^n (X_i·w)(X_{in})  \\end{bmatrix} = \n",
    "\\frac{2}{m}X^TXw $\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PCA by Gradient Ascend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.empty(( 100, 2))\n",
    "X[:, 0] = np.random.uniform(0., 100., size = 100)\n",
    "X[:, 1] = 0.75 * X[:, 0] + 3. + np.random.normal(0, 10, size = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD4CAYAAAAJmJb0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAbYElEQVR4nO3df6zddX3H8eeb2wve4o9bRiHlArbbSJnioO7GOe/ilDqLwKQxQzFhqcrSLDETiKnezmXoH8ab1DhM5pY06EYmUxBY6cRZWStZRgLzlhYBa5UJlN5WuM5eNHAHt+W9P8455fT2+z3nfM/39+e8HknTe7/nx/f7OdD393Pen/fn8zF3R0REwnRK2RcgIiL5UZAXEQmYgryISMAU5EVEAqYgLyISsCVlX0C7M88801euXFn2ZYiI1Mru3bt/4e7Lox6rVJBfuXIl09PTZV+GiEitmNnTcY8pXSMiEjAFeRGRgCnIi4gETEFeRCRgCvIiIgGrVHWNiEhotu2ZYcuO/Ryam+ec0RE2rVvN+jVjhZ1fQV5EJCfb9syw+e5HmV84BsDM3Dyb734UoLBAryAvIpKTLTv2Hw/wLfMLx9iyY//xIJ93T19BXkQkJ4fm5jseL6Knr4FXEZGcnDM60vF4p55+VhTkRURysmndakaGh044NjI8xKZ1q4HuPf0sKMiLiORk/ZoxvvCBtzA2OoIBY6MjfOEDbzmeiunW08+CcvIiIhGyGhBdv2Ys9nWb1q0+IScPJ/b0s6AgLyKlKLt+vJOsBkS7tbH1c56fg7l7Zm+W1vj4uGupYZHwLQ6i0OjBtqcyyjQxtYuZiLz42OgID0xe2tN7FNlGM9vt7uNRjyknLyKFK6KqJI0sBkSr0kala0SkcEVUlaRxzuhIZE8+yYBoXFtm5uaZmNpVWJpKPXkRKVwRVSXb9swwMbWLVZP3MjG1i217Znp+bbfSx17EtcVoBHrn1Vx/kmtLSkFeRAqXRRDtpJUP7zeYdit97EVUGw1YPAqadwpH6RoRKVzeVSW9rBnTyzWmuZ6oNkalgCDfNFUmQd7MbgT+nMZN6lHgo8BS4HZgJfAU8EF3P5LF+USk/tIG0U56WTOmiPLNxW2Mq9rJMk21WOp0jZmNAZ8Axt39ImAIuAaYBHa6+wXAzubvIiK565TzT5vKSSPvNFWUrHLyS4ARM1tCowd/CLgKuLX5+K3A+ozOJSLSUadgWlZpY+vbw/zCMYbMgP5y/UmlDvLuPgN8ETgAHAaed/fvAWe7++Hmcw4DZ0W93sw2mtm0mU3Pzs6mvRwRkY4Dp2WUb7Z/ewA45n78ppP35K/UOXkzW0aj174KmAO+ZWbX9vp6d98KbIXGjNe01yMiAvE5/yxq4JPKYiC4X1mka94DPOnus+6+ANwNvAN41sxWADT/fi6Dc4mIpFJGXrzMyV9ZBPkDwNvNbKmZGbAW2AdsBzY0n7MBuCeDc4mIpJJFDXxSRUz+ipM6XePuD5nZncDDwFFgD430y2uBO8zsOho3gqvTnktEJAt5lm9GKWJJ4TiZ1Mm7+03ATYsOv0SjVy8iMtCKWFI4jma8iogUoOhvDy0K8iIiJShq1q2CvIgEq6q7T2W181QvFORFpC9VDaAtRQbSpIqsm9dSwyKSWJnrv/SqKjszRSmybl5BXkQSq3IAbany7lNF1s0rXSMiiVUxgC5OH71hZJi5+YWTnlfEBKRuiqybV5AXkcTKWP+lk6j8+/CQMXyKsfDKq0tiFTUBqZsi6+YV5EUksTJncEaJSh8tHHOWLR1m6alLKjk4XFTdvIK8iCRW5gzOKHFporkXF9jzN+8FXk3n3Hj73tKvt0gK8iLSl7JmcEbplj6qcjll3lRdIyK112354DpUA+VFPXkRqb1u6aMqVgMVRUFeRILQKX1UtWqgIildIyKl2rZnhompXayavJeJqV25zJotYzeoqlBPXkRK02lAFLKr3qlaNVCRzL06e2ePj4/79PR02ZchIgWZmNoVmUZZtnSY/1t45aQ6/Ly36asrM9vt7uNRj6knLyJ9S7sSZdzA55EXT16OIK9VGkOnnLyI9CWLlSiTDnwOQjVM1hTkRaQvWdSexw2Ijo4MRz5/EKphsqZ0jYj0JYva87gBUaBSa+PUmYK8iMTqlHPPqva8U337IFbDZE1BXkQidVvvJe+VKKu0Nk6dKScvIpG65dzXrxnjCx94C2OjIxgwNjqiEscKUk9eRCL1knMf5N521Tcyb1FPXkQiFbkPad3UYSPzFgV5EYk0yOu9dFOnpYuVrhGRSIO83ks3dVq6OJMgb2ajwC3ARYADHwP2A7cDK4GngA+6+5EszicixRjknHsndVq6OKt0zZeB77r7hcDFwD5gEtjp7hcAO5u/i4jkpohli6FeqazUPXkzez3wTuAjAO7+MvCymV0FvKv5tFuB+4FPpz2fiEiUIvdxrVMqK/VSw2Z2CbAV+BGNXvxu4Hpgxt1H2553xN2XRbx+I7AR4Pzzz/+9p59+OtX1iMhgilu2eGx0hAcmLy3hiorTaanhLNI1S4C3Av/g7muAF0iQmnH3re4+7u7jy5cvz+ByRMJSVAqi7uo0GFqkLIL8QeCguz/U/P1OGkH/WTNbAdD8+7kMziUyUOpUj1021fVHSx3k3f3nwDNm1hpxWEsjdbMd2NA8tgG4J+25RAZNneqxy1anwdAiZVUn/5fAbWZ2KvAz4KM0biB3mNl1wAHg6ozOJTIwlILoXZ0GQ4uUSZB3971AVNJ/bRbvLzJoWuuixJVFDHoKIo7q+k+mGa8iFbO4FHCxrFIQdVlgK0uD2GYFeZGKicrDt4xlFJi61ZSHGAyLrKOvEgV5kYqJy7cbZFbv3W1AN00wrOoNolObq3B9edEqlCIVU0QpYKcB3TQVPVUu+RzUQWwFeZGKKaIUsNONJE0wrHLJ56DW0SvIi1RMEdvqdbqRpAmGVe4tD2odvXLyIhWUdylgt5rypBt016Hkc1Dr6FMvUJal8fFxn56eLvsyRI6r6iBi3pK0u5eST23wna9OC5SpJy8SY1BL7iDZN4kiSj6zMKg3bAV5kRiDWnKXVBEln2kN8g1bA68iMao8iFgldahaqXLVT94U5EVi1CF4VUEdqlYG+YatdI1IjE3rVieuMqmSLHPQnd6rDlUrddp4O2sK8jIQ+gl4dQhecXrNQffyufTyXlVf/bHuN+w0VEIpwYsq8Qu9rK+X/U57/VxC2Ts15OoalVDKQBvEKplectC9fi6h5LOr/m0jLwryEryqB6m4HmaanmcvOeheP5dBzmeHQNU1ErwqV8nErdr419seTbWaYy8VL71+LnWonpF4CvISvCoHqbiUyTceeiZVXXcvi5z1+rkUsWCa5EfpGglelatk4lImx2IKIqKeH5fW6ZaDTvK5DGo+OwQK8jIQqhqk4vLdQ2aRgX5xKiXtdP2qfi6SHaVrREoUlzL58O+f11MqZZCn60tv1JMXKVGnlMn4G8/omkqpeuWQlE9BXqRkcSmTXlIpKm+UbpSukVxt2zPDxNQuVk3ey8TUrkps6BySKlcOSTWoJy+5GeQ1vItS5cohqQYFecnNIC4nUAZVyEgnmQV5MxsCpoEZd7/SzM4AbgdWAk8BH3T3I1mdT6qv6oOCIS9YJdKSZU7+emBf2++TwE53vwDY2fxdBkgdlxPQmIGEJpMgb2bnAlcAt7Qdvgq4tfnzrcD6LM4l9RE1KAjwwktHSw+maevLNaAsdZFVuuZm4FPA69qOne3uhwHc/bCZnZXRuaQmWqmPz/3b4xx5ceH48bn5hdIHYNOkkjSgLHWSuidvZlcCz7n77j5fv9HMps1senZ2Nu3lSMWsXzPG0lNP7ktkPSszac86TSop7lvAJ+94RD16qZwsevITwPvN7HLgNcDrzezrwLNmtqLZi18BPBf1YnffCmyFxs5QGVyPVEzWA7CLB0zffeFy7to9k6hnnWY7uE6LiqlHL1WTuifv7pvd/Vx3XwlcA+xy92uB7cCG5tM2APekPZfUU5YDsFEDprc9eCBxfj3N8rmdrls9eqmaPOvkp4A7zOw64ABwdY7nkgrr1mtOUsoYlSqJ+/rX7ZtCv/XlUe1ppx69VEmmQd7d7wfub/78v8DaLN9f6qnTrMykg5hJUjx5lWq2ruvGO/YSs+y7Jn1JZWjGqxQirtecdFZs3IJcxok9+iLWb1lixkJclKdxw5qY2pVospUmaEnWtECZlCrpoGxU7b0B7/itMwrdnm7Ljv0svNK5TsAg0WQrTdCSPKgnL6VKulTu+jVjTD/9S2578MDxnrsDDx94vtB9R7uljRZ/s4ATB4Ojeuta60fyoJ68lKqfpXK//+PZjgG0CJ3y/WOjI7GDwa3eeVRvvepr/Ug9KchLqfopZUw7WzWL5Qiibk7DpxjLlg5zaG6eIbPI1w2ZxfbWq7zWj9SX0jVSiE4DiklLGfvdDSnL5QgWVwy9YWSYF14+enz5hqhNuEeGh2LLLg/NzfO3H7qk7wlaInHUk5fcZT2g2O9uSFlver1+zRgPTF7Kk1NXcPppS1g4dnJgHzI74RvKWIfeepoJWiJx1JOX3HULrklLBvvdDSnPnHfce7zizpNTVwCNm90LLx096TntNyhtACJZU5CX3MUFwFaPvp/0ST/BMM9Nr7u99+JUUcuypcPc9CdvVmCX3ChdI7mLC6KdBiHzkOem193eO+rbDMDSU5cowEuuFOQld3EBMGpwEvIrGcwz591679GR4ePHXjP86j8vlUdKWZSukdzF5dC37NifW/qk07Xk2XN+6egrx38+8uKrm6PkmSoS6URBXnIRVTL5wOSlJz0vpJLBTgPMadavF0lD6RrJXK8lk4vTJ8uWDnPaklO48fa9tdw3tVNKRuWRUhb15CVzSdZgaaVPQtg3tVtKRuWRUgb15CVz/QwyZj1RqQx5Vu+I9Es9+Qqq+5ri/Qwy9nNjqNrn1O8kLZE8KcgvUnbgCCFt0c8gY9IbQ1U/J6VkpGqUrmlThU0bQkhb9DPImDTVEcLnJFIE9eTbVGHThlAmzSTt0SZNdYTyOZX9zVHCpyDfpgqBY5AnzSS5MYTwOVU15SRhUbqmTRU2bVCFRm9C+JyUcpIiqCffpp8Bw6y/bodWoZFXOiKEz6kK3xwlfArybZIGjry+bkelLeqYu807HVH3SpYQUk5SfQryiyQJHEUN1NY1d1uFgewq03o2UgTl5FMo6ut2XXO3Skd0pvVspAjqyadQ1NftLILltj0zfHb748zNNzaaLmJHIqUjuqt7ykmqTz35RbbtmWFiaherJu/tuhJiURUeaat+tu2ZYdO3Hjke4KGx1vmmOx/JdaJXCBUwInWXOsib2Xlm9n0z22dmj5vZ9c3jZ5jZfWb20+bfy9Jfbr6Szngt6ut22mC5Zcd+Fl45eRemhWOea8on1HREko6ASNnMY7Zg6/kNzFYAK9z9YTN7HbAbWA98BPilu0+Z2SSwzN0/3em9xsfHfXp6OtX1pDExtSsyvTA2OhK54UUaSatl0lTXrJq8l7j/ygY8OXVF8gYMqKgNuUeGh4K4eUl9mdludx+Peix1Tt7dDwOHmz//2sz2AWPAVcC7mk+7Fbgf6Bjky5b1QGFcYO6nWiZN7jYuN956THqniiGpm0wHXs1sJbAGeAg4u3kDwN0Pm9lZMa/ZCGwEOP/887O8nMS6DRQm6U13CuRFB4pN61az6VuPnJSyGR6y3PPjdazv70QVQ1I3mQ28mtlrgbuAG9z9V72+zt23uvu4u48vX748q8vpS6fcd9J8fadAXnSgWL9mjC1XX8zoyPDxY8uWDrPlTy/ONeBWYVXPrFVh6QuRJDLpyZvZMI0Af5u73908/KyZrWj24lcAz2Vxrjx1mvE6MbUrMmjfcPtebrh970kliZ0CeRmlhVmV6iXpmYeY2tAEJqmb1EHezAz4KrDP3b/U9tB2YAMw1fz7nrTnKkJcMOzWy26VJLbeo1MgzzpQFJUSSTqWEGJqI4Q1c2SwZNGTnwD+DHjUzPY2j/0VjeB+h5ldBxwArs7gXF3lFfA6DV62tEoS168Z6xjIswwURS55kLRnHupkKE1gkjrJorrmv2hU4kVZm/b9k8gz4EUF7SitXmq3QJ5VoCgyJZK0Z67Uhkj5glrWIM+A1x60O/Xo23upRfT4ikyJJO2ZK7UhUr6ggnxWa7x06323lgkooyRxsbjA6zQmd7Vff9pUVj89c6U2RMoVVJBPmwPuNd3T+rnoBb+idEojtV8/kDqVpZ55+UKbdyD5S72sQZbSLmuQdsp5kcsatEv7D7f1+rg00ljzJldG2yQ7WlJB4nRa1iCoVSjTLohVRslfFhOG1q8Z44HJS2NHvw/NzQdZzjho6rqvgJQrqHQN5LPGS54lf1kOFne7/hDLGQeJbtTSj6B68mn1s6Rv2mVns/yH2+n6tbZ7/WlJBelHcD35NMrYyDvLbw+9XL8G7epL8w6kH0ENvBYti4FaDaZJEqqukSi5ric/yLJItagsUZLQvANJSkG+qZ8eUlapFv3DFZG8aOCV/ssYNZgpIlWnnjz9lzHWLdWifK7I4AkiyCfdlm/xc9Pk1uuSailqSWLdSESqpfbpmiSplrjnji4dPum5EFb9cRGzJUPc7k+k7mrfk+8WvNp7lS++fDTyuactOYWR4aGg64+LmC0Z4nZ/InVX+558XJBq9SLbe5VHXlyIfO7z8wup1rypgyJmS2ravUj11L4nP7p0ODZ4d9vFqeWc0ZHa5Nb7VcRsyVC3+xOps9r35NNO2B0ZHuLdFy5Ptf5MHaRdobMXKikVqZ7a9+Sfn4/uxccZHRnm9NOWHM/Tv/vC5dy1eya3qpMqVZvk/W2lbiWlIoOg9kE+LkUQZWR4iM++/8TdmyamduU2WFhU2WKVhJ72Eqmb2qdrolIE7YbMOqYn8hws1CYPIlK22vfko/ZbbellNcc8BwtVbSIiZat9Tx4agX7vTe/l5g9dknhgMc/BQm3yICJlq31Pvl0/+eA8Bwu1yYOIlC2oIN+vvAYLVW0iImVTkM+Zqk1EpExB5ORFRCRa7j15M7sM+DIwBNzi7lNZn6NKE46qfE0iMnhyDfJmNgR8Bfhj4CDwAzPb7u4/yuocZUw46hbAB3ESlIhUU97pmrcBT7j7z9z9ZeCbwFVZniDJhKNte2ZSr1HTy5rpmgQlIlWRd5AfA55p+/1g89hxZrbRzKbNbHp2djbxCXqdcJTVhha9BHBNghKRqsg7yFvEsRPWjXT3re4+7u7jy5cvT3yCXiccZdW77iWAaxKUiFRF3kH+IHBe2+/nAoeyPEGvM1a7BedeUzm9BHAtuSsiVZF3kP8BcIGZrTKzU4FrgO1ZnqDXddI7BeckqZxeAngRa7eLiPTCPO2uG91OYHY5cDONEsqvufvn4547Pj7u09PTuVzH4ooXeHUBsy079kcuUjY2OsIDk5dGvpfKI0WkKsxst7uPRz2We528u38H+E7e5+mm0xIDN96+N/I1cSkezWIVkboYqGUN4oKz9iYVkVBpWQM0UCoi4RqonnwcrRYpIqFSkG9Snl1EQqR0jYhIwBTkRUQCpiAvIhIwBXkRkYApyIuIBExBXkQkYAryIiIBU5AXEQmYgryISMAU5EVEAqYgLyISMAV5EZGAKciLiARMQV5EJGAK8iIiAVOQFxEJmIK8iEjAFORFRAKmIC8iEjAFeRGRgCnIi4gETEFeRCRgCvIiIgFbUvYFZGXbnhm27NjPobl5zhkdYdO61axfM1b2ZYmIlCpVT97MtpjZj83sh2b2r2Y22vbYZjN7wsz2m9m61FfawbY9M2y++1Fm5uZxYGZuns13P8q2PTN5nlZEpPLSpmvuAy5y998FfgJsBjCzNwHXAG8GLgP+3syGUp4r1pYd+5lfOHbCsfmFY2zZsT+vU4qI1EKqIO/u33P3o81fHwTObf58FfBNd3/J3Z8EngDeluZcnRyam090XERkUGQ58Pox4N+bP48Bz7Q9drB57CRmttHMps1senZ2tq8TnzM6kui4iMig6Brkzew/zOyxiD9XtT3nM8BR4LbWoYi38qj3d/et7j7u7uPLly/vpw1sWreakeETs0Ejw0NsWre6r/cTEQlF1+oad39Pp8fNbANwJbDW3VuB/CBwXtvTzgUO9XuR3bSqaFRdIyJyolQllGZ2GfBp4I/c/cW2h7YD/2JmXwLOAS4A/jvNubpZv2ZMQV1EZJG0dfJ/B5wG3GdmAA+6+1+4++NmdgfwIxppnI+7+7EO7yMiIjlIFeTd/bc7PPZ54PNp3l9ERNLRsgYiIgFTkBcRCZiCvIhIwOzVqsfymdks8HSfLz8T+EWGl1MHavNgUJsHQ5o2v9HdIycaVSrIp2Fm0+4+XvZ1FEltHgxq82DIq81K14iIBExBXkQkYCEF+a1lX0AJ1ObBoDYPhlzaHExOXkREThZST15ERBZRkBcRCVjtg7yZXdbcR/YJM5ss+3ryYGbnmdn3zWyfmT1uZtc3j59hZveZ2U+bfy8r+1qzZmZDZrbHzL7d/D3oNpvZqJnd2dw7eZ+Z/cEAtPnG5v/Xj5nZN8zsNSG22cy+ZmbPmdljbcdi25nVPtm1DvLNfWO/ArwPeBPw4eb+sqE5CnzS3X8HeDvw8WY7J4Gd7n4BsLP5e2iuB/a1/R56m78MfNfdLwQuptH2YNtsZmPAJ4Bxd78IGKKxP3SIbf4nGntet4tsZ5b7ZNc6yNPYN/YJd/+Zu78MfJPG/rJBcffD7v5w8+df0/iHP0ajrbc2n3YrsL6UC8yJmZ0LXAHc0nY42Dab2euBdwJfBXD3l919joDb3LQEGDGzJcBSGhsMBddmd/9P4JeLDse1M7N9suse5HveSzYUZrYSWAM8BJzt7oehcSMAzirx0vJwM/Ap4JW2YyG3+TeBWeAfmymqW8zsdAJus7vPAF8EDgCHgefd/XsE3OZF4tqZWWyre5DveS/ZEJjZa4G7gBvc/VdlX0+ezOxK4Dl33132tRRoCfBW4B/cfQ3wAmGkKWI1c9BXAato7CJ3upldW+5VVUJmsa3uQb7QvWTLZGbDNAL8be5+d/Pws2a2ovn4CuC5sq4vBxPA+83sKRppuEvN7OuE3eaDwEF3f6j5+500gn7IbX4P8KS7z7r7AnA38A7CbnO7uHZmFtvqHuR/AFxgZqvM7FQaAxXbS76mzFljb8WvAvvc/UttD20HNjR/3gDcU/S15cXdN7v7ue6+ksZ/113ufi1ht/nnwDNmtrp5aC2NLTSDbTONNM3bzWxp8//ztTTGnEJuc7u4dm4HrjGz08xsFWn2yXb3Wv8BLgd+AvwP8JmyryenNv4hja9qPwT2Nv9cDvwGjRH5nzb/PqPsa82p/e8Cvt38Oeg2A5cA083/1tuAZQPQ5s8BPwYeA/6Zxr7RwbUZ+AaNcYcFGj316zq1E/hMM67tB97X73m1rIGISMDqnq4REZEOFORFRAKmIC8iEjAFeRGRgCnIi4gETEFeRCRgCvIiIgH7fyBExKRzTgrXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X[:, 0], X[:, 1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### demean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def demean(X):\n",
    "    return X - np.mean(X, axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_demean = demean(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD4CAYAAAAJmJb0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAbbElEQVR4nO3df6xcdZnH8ffD5VJvdfWWAEpv6bbrliqIWjOyZm820SoURaExGmuiaVaSBoPrj2i1lY26fxBv0kTdRP2jURMS2QVc2dIVTQWqmywR8JbCsgUrrETgFgV/VI29C7fl2T/mzGV6e87MnDm/v/N5JaR3zsydOWfCfeaZ5/t8v19zd0REJEynVX0CIiJSHAV5EZGAKciLiARMQV5EJGAK8iIiATu96hPodtZZZ/maNWuqPg0RkUY5cODAb9z97Lj7ahXk16xZw+zsbNWnISLSKGb2y6T7VK4REQmYgryISMAU5EVEAqYgLyISMAV5EZGA1aq7RkQkNHsOzrFr32GOHJ1n5eQE2zetZ/OGqdJeX0FeRKQgew7OsfOWB5lfOAHA3NF5dt7yIEBpgV5BXkSkILv2HV4M8B3zCyfYte/wYpAvOtNXkBcRKciRo/M9j5eR6WvgVUSkICsnJ3oe75Xp50VBXkSkINs3rWdifOykYxPjY2zftB7on+nnQUFeRKQgmzdM8cV3X8TU5AQGTE1O8MV3X7RYiumX6edBNXkRkRh5DYhu3jCV+HvbN60/qSYPJ2f6eVCQF5FKVN0/3kteA6L9rrHzc5Hvg7l7bk+WVavVci01LBK+pUEU2hlsdymjStMz+5mLqYtPTU5w146NAz1HmddoZgfcvRV3n2ryIlK6MrpKsshjQLQu16hyjYiUroyukixWTk7EZvJpBkSTrmXu6DzTM/tLK1MpkxeR0pXRVbLn4BzTM/tZu+M2pmf2s+fg3MC/26/1cRBJ12K0A73zQq0/zbmlpSAvIqXLI4j20qmHDxtM+7U+DiLuGg1YOgpadAlH5RoRKV3RXSWDrBkzyDlmOZ+4a4wrAUGxZSoFeRGpRNYg2ssga8aU0b659BqTunbyLFMtpXKNiASnV80/aykni6LLVHEU5EUkOL2CaVWtjZ1vD/MLJxgzA4ar9aelIC8iwek1cFpF+2b3tweAE+6LHzpFT/5STV5EgpRU88+jBz6tPAaCh6VMXkRGShV18Sonf+UW5M1szMwOmtn3ottnmtntZvZI9O+KvF5LRGRYefTAp1XG5K8keZZrPgY8DLw0ur0DuNPdZ8xsR3T7Mzm+nojIUIps34xTxpLCSXLJ5M1sFXA58I2uw1cC10c/Xw9szuO1RESapopvDx15ZfJfAT4N/EXXsZe7+1MA7v6UmZ0T94tmtg3YBrB69eqcTkdEpF7K/vbQkTnIm9k7gafd/YCZvTnt77v7bmA3tNeTz3o+IiJNUNas2zwy+WngCjN7B/Ai4KVm9m3g12Z2bpTFnws8ncNriYgMrK67T+W189QgMtfk3X2nu69y9zXAFmC/u38A2AtsjR62Fbg162uJSH1kWcq3DFUuX9BPmbNui+yTnwEuMbNHgEui2yISgDoH0I667MwUp8y++VyDvLv/2N3fGf38W3d/q7uvi/79XZ6vJSLVqXMA7ajz7lNl9s1rWQMRSa2OAXRp/f1lE+McnV845XFlTEDqp8y+eQV5EUmtivVfeokbyBwfM8ZPMxaef6Fpr6wJSP0UvWlKNwV5EUmtyhmcceLKRwsnnBXLx1l+xum1666B8vrmFeRFJLUyM9FBJJWJjh5b4ODnLgVeKOd84qb7Kz/fMinIi8hQqprBGadf+ajMvvS60VLDItJ4/ZYPbkI3UFGUyYtI4/UrH9WxG6gsCvIiEoRe5aO6dQOVSeUaEalUGcsjVLEbVF0okxeRyvQaEIX8unfq1g1UJnOvz+q+rVbLZ2dnqz4NESnJ9Mz+2DLKiuXj/N/C86f04Ze10UbTmNkBd2/F3adMXkSGlnUp36SBz98fO3U5gk43jIJ8OqrJi8hQ8liJMu3A5yh0w+RNQV5EhpJH73nSgOjkxHjs40ehGyZvKteIyFDy6D1PGhAFarU2TpMpyItIol4197x6z3v1t49iN0zeFORFJFa/9V6KXomyTmvjNJlq8iISq1/NffOGKb747ouYmpzAgKnJCbU41pAyeRGJNUjNfZSz7azto2VRJi8iscrch7RpmrCReYeCvIjEGuX1Xvpp0tLFKteISKxRXu+lnyYtXawgLyKJRrnm3kuTli5WuUZEglHGssXQrFKWMnkRCUKZ+7g2qZSlIC8iQeg1GFpE8G1KKUtBXqTmmtKPXbUmDYaWSTV5kRprUj921dTXHy9zkDez88zsR2b2sJkdMrOPRcfPNLPbzeyR6N8V2U9XZLQ0qR+7ak0aDC1THpn8ceCT7v5q4E3ANWZ2AbADuNPd1wF3RrdFJAWVIAantXTiZa7Ju/tTwFPRz38ys4eBKeBK4M3Rw64Hfgx8JuvriYyCTh0+aQfmUS9BJGnKYGiZch14NbM1wAbgHuDl0QcA7v6UmZ2T8DvbgG0Aq1evzvN0RBppaSvgUnmVIEZxQHcUrzm3IG9mLwG+C3zc3f9oZgP9nrvvBnYDtFqtpMRFZGTE1eE7pnIKTP16ykMMhmX20ddJLkHezMZpB/gb3P2W6PCvzezcKIs/F3g6j9cSCV1Svd2Au3ZszOU1+g3oZgmGdf2AKLuPvi7y6K4x4JvAw+7+pa679gJbo5+3ArdmfS2RUVBGK2CvAd0sHT11bvkc1UHsPLprpoEPAhvN7P7ov3cAM8AlZvYIcEl0W0T6KKMVsNcHSZZgWOeWz1Hto88c5N39v9zd3P217v766L/vu/tv3f2t7r4u+vd3eZywSOjKaAXs9UGSJRjWOVse1T56LWsgUkNFtwL2W2Ar7QbdTWj5bNKiYnky9/o0tLRaLZ+dna36NEQW1XUQsWhprnuQlk9NSiqWmR1w91bcfcrkRRKMassdpPsmUUbLZx5G9QNbQV4kwai23KVVRstnVqP8ga1VKEUS1HkQsU6a0LVS566foinIiyRoQvCqgyZ0rYzyB7bKNSIJtm9an7rLpE7yrEH3eq4mdK00aePtvCnIy0gYJuA1IXglGbQGPcj7Mshz1X31x6Z/YGehFkoJXlyLX+htfdMz+2Mz16nJicXB0EHfl0GeqwlC7q5RC6WMtFHskhmkBj3o+xJKPbvu3zaKoiAvwat7kErKMLNknoPUoAd9X0a5nh0CdddI8OrcJZO0auM/7nkw02qOg3S8DPq+NKF7RpIpyEvw6hykkkom/3rPE5n6ugdZ5GzQ90V7pzabyjUSvDp3ySSVTE4kNETEPT6prNOvBp3mfRnVenYIFORlJNQ1SCXVu8fMYgP90lJK1un6dX1fJD8q14hUKKlk8v6/OW+gUsooT9eXwSiTF6lQr5JJ6y/P7FtKqXvnkFRPQV6kYkklk0FKKWpvlH5UrpFC7Tk4x/TMftbuuI3pmf212NA5JHXuHJJ6UCYvhRnlNbzLUufOIakHBXkpzCguJ1AFdchILwryUpi6DwqGvGCVSIdq8lKYJi4noDEDCY2CvBQmblAQ4M/PHq88mGbtL9eAsjSFyjVSmE7p45/+4xC/P7awePzo/ELlA7BZSkkaUJYmUSYvhdq8YYrlZ5yaS+Q9KzNtZp2llJT0LeCTNz+gjF5qR5m8FC7vAdilA6ZvedXZfPfAXKrMOst2cL0WFVNGL3WjTF4Kl+cAbNyA6Q13P566vp5l+dxe562MXuqm8EzezC4D/hkYA77h7jNFv6bUS7+sOU0rY1ypJGmX4n7fFIbtL4+7nm7K6KVOCs3kzWwM+BrwduAC4P1mdkGRryn10ytrTtvKmKbEU1SrZud6zJIfo5UgpS6KzuQvBh51918AmNmNwJXAQwW/rtRMUtacdlZs0oJcxskZfRnrt5xuxkLC5h7Q/sCantmfarKVJmhJ3oquyU8BT3TdfjI6tsjMtpnZrJnNPvPMMwWfjtRN2kHZuN57A/72lWeWuj3drn2HWXg+OcB3zivNZCtN0JIiFJ3Jx32hPekvw913A7sBWq1W778aCU7apXI3b5hi9pe/44a7H1/8H8mB+x7/Q6n7jvYrGy39ZgEnl3DisnWt9SNFKDqTfxI4r+v2KuBIwa8pDTLMUrk/+tkzPQNoGXrV+6cmJxIHgzvZeVy2Xve1fqSZig7yPwXWmdlaMzsD2ALsLfg1pUGGaWXMOls1j+UI4j6cxk8zViwf58jRecYSRmXHzBKz9Tqv9SPNVWi5xt2Pm9lHgH20Wyi/5e6HinxNqadeA4ppWxmH3Q0pz+UIlq7j/rKJcf783PHF5RviNuGeGB9LbLs8cnSeL7/v9UNP0BJJUvhkKHf/vruf7+6vdPfrin49qZ+8BxSH3Q0p702vN2+Y4q4dG3ls5nJevOx0Fk6cGtjHzE76hjLVI1vPMkFLJImWNZDC9QuuaVsGh90Nqciad9JzPO/OYzOXA+0Puz8/e/yUx3R/QGkDEMmbgrwULikAdjL6YconwwTDIje97vfcS0tFHSuWj/P5d12owC6F0do1UrikINprELIIRW563e+5477NACw/43QFeCmUgrwULikAxg1OQnEtg0XWvDvPPTkxvnjsReMv/HmpPVKqonKNFC6phr5r3+HCyie9zqXIzPnZ488v/vz7Yy9sjlJkqUikFwV5KURcy+RdOzae8riQWgZ7DTBnWb9eJAuVayR3g7ZMLi2frFg+zrLTT+MTN93fyH1Te5Vk1B4pVVEmL7lLswZLp3wSwr6p/Uoyao+UKiiTl9wNM8iY90SlKhTZvSMyLGXyNdT0NcWHGWQc5oOhbu/TsJO0RIqkIL9E1YEjhLLFMIOMaT8Y6vo+qSQjdaNyTZc6bNoQQtlimEHGtKWOEN4nkTIok+9Sh00bQpk0kzajTVvqCOV9qvqbo4RPQb5LHQLHKE+aSfPBEML7VNeSk4RF5Zouddi0QR0agwnhfVLJScqgTL7LMAOGeX/dDq1Do6hyRAjvUx2+OUr4FOS7pA0cRX3djitbNLF2W3Q5oumdLCGUnKT+FOSXSBM4yhqobWrttg4D2XWm9WykDKrJZ1DW1+2m1m5VjuhN69lIGZTJZ1DW1+08guWeg3N8Ye8hjs63N5ouY0cilSP6a3rJSepPmfwSew7OMT2zn7U7buu7EmJZHR5Zu372HJxj+3ceWAzw0F7rfPu/PVDoRK8QOmBEmk5BvkvaGa9lfd3OGix37TvMwvOn7sK0cMILLfmEWo5IkwiIVE3lmi7DDBQO+3U7TbdM1nbBXmWdouvjoZUjmjoILqNLQb5L3gOFSYF8mECRJVgm1cY798ng1DEkTaMg36XfQGGa7LtXIC87UGzftJ7t33nglJLN+JgVXh9vYn9/L+oYkqZRTb5Lr9p32np9r0BedqDYvGGKXe99HZMT44vHViwfZ9d7XldowK3Dqp55q8PSFyJpKJPv0qv2PT2zPzZof/ym+/n4Tfef0pLYK5BX0VqYV208TWYeYmlDE5ikaRTkl0gKhv2y7E5LYuc5egXyvANFWSWRtGMJIZY2QlgzR0ZLpiBvZruAdwHPAf8L/L27H43u2wlcBZwAPuru+7Kd6mCKCni9Bi87Oi2JmzdM9QzkeQaKMrs90mbmoU6GCq1jSMKWtSZ/O/Aad38t8HNgJ4CZXQBsAS4ELgO+bmZjic+SkyJrwHH1+jidLLVfj/jmDVPctWMjj81czl07Ng4dNMpc8iBtZq7JUCLVy5TJu/sPu27eDbwn+vlK4EZ3fxZ4zMweBS4GfpLl9fopsgbcnX33yui7s9QyMr4ySyJpM3OVNkSql2dN/kPATdHPU7SDfseT0bFTmNk2YBvA6tWrM51AXmu8JAWlTtDuLBNQRUviUkmB14Hpmf0nnX/WUtYwYwkqbYhUq2+QN7M7gFfE3HWtu98aPeZa4DhwQ+fXYh5/6rx6wN13A7sBWq1W7GMGlbUGPGh9u/Nz2Qt+xYkLvB3d5w9krt0rM69eaPMOpHjmnimuYmZbgauBt7r7sejYTgB3/2J0ex/wBXfvWa5ptVo+Ozs79LksDdLQzjQHXS9lemZ/7IfE1OQEd+3YOPR59ZP1D7fz+0llpKnoQ66Ka5P8ZP3/W8JlZgfcvRV3X6aBVzO7DPgMcEUnwEf2AlvMbJmZrQXWAfdmea1BZF0Qq4qWvzwGizuDuHFfn6B9/iG2M46apu4rINXKWpP/KrAMuN3MAO5296vd/ZCZ3Qw8RLuMc427n1pPKEARa7wU2fKX52Bxv/MPsZ1xlOiDWoaRKZN397929/Pc/fXRf1d33Xedu7/S3de7+w+yn2rxhmn5y7rsbJ5/uL3OX+2MzaclFWQYmvHapYqNvPP89jDI+WvQrrm0pIIMI/PAa56yDryWLY+BWg2mSRrqrpE4vQZelclnkEepRW2JkobmHUhaCvKRYTKkvEot+sMVkaJoPXmGb2PUYKaI1J0yeYZvY2xaqUX1XJHRE0SQT7st39LHZqmtN6XUUtaSxPogEamXxpdr0pRakh47uXz8lMdCWP3HZcyWDHG7P5Gma3wm3y94dWeVx547HvvYZaefxsT4WND9x2XMlgxxuz+Rpmt8Jp8UpDpZZHdW+ftjC7GP/cP8QqY1b5qgjNmSmnYvUj+Nz+Qnl48nBu+45XfjrJycaExtfVhlzJYMdbs/kSZrfCafdcLuxPgYb3nV2ZnWn2mCrCt0DkItpSL10/hM/g/z8Vl8ksmJcV687PTFOv1bXnU23z0wV1jXSZ26TYr+ttK0llKRUdD4IJ9UIogzMT7GF644efem6Zn9hQ0WltW2WCehl71Emqbx5Zq4EkG3MbOe5YkiBwu1yYOIVK3xmXzcfqsdg6zmWORgobpNRKRqjc/koR3o7//8pXzlfa9PPbBY5GChNnkQkao1PpPvNkw9uMjBQm3yICJVCyrID6uowUJ1m4hI1RTkC6ZuExGpUhA1eRERiRdEJl+nCUd1PicRGT2ND/JVTDjqF8BHcRKUiNRT48s1aSYc7Tk4l3mNmkHWTNckKBGpi8YH+UEnHOW1ocUgAVyToESkLhof5AedcJRXdj1IANckKBGpi8YH+UFnrPYLzoOWcgYJ4FpyV0TqovFBftB10nsF5zSlnEECeBlrt4uIDMI8664bgJl9CtgFnO3uv4mO7QSuAk4AH3X3ff2ep9Vq+ezsbObzibO04wVeWMBs177DsYuUTU1OcNeOjbHPpfZIEakLMzvg7q24+zK3UJrZecAlwONdxy4AtgAXAiuBO8zsfHcfbD++AvRaYuATN90f+ztJJR7NYhWRpsijT/7LwKeBW7uOXQnc6O7PAo+Z2aPAxcBPcni9oSUFZ+1NKiKhylSTN7MrgDl3f2DJXVPAE123n4yOxT3HNjObNbPZZ555JsvpDE0DpSISqr6ZvJndAbwi5q5rgc8Cl8b9Wsyx2OK/u+8GdkO7Jt/vfIqg1SJFJFR9g7y7vy3uuJldBKwFHjAzgFXAfWZ2Me3M/byuh68CjmQ+2wKpzi4iIRq6XOPuD7r7Oe6+xt3X0A7sb3D3XwF7gS1mtszM1gLrgHtzOWMRERlYIQuUufshM7sZeAg4DlxTZWeNiMioyi3IR9l89+3rgOvyen4REUmv8TNeRUQkmYK8iEjAFORFRAKmIC8iEjAFeRGRgCnIi4gETEFeRCRgCvIiIgFTkBcRCZiCvIhIwBTkRUQCpiAvIhIwBXkRkYApyIuIBExBXkQkYAryIiIBU5AXEQmYgryISMAU5EVEAqYgLyISsNw28q7anoNz7Np3mCNH51k5OcH2TevZvGGq6tMSEalUEEF+z8E5dt7yIPMLJwCYOzrPzlseBFCgF5GRFkS5Zte+w4sBvmN+4QS79h2u6IxEROohiCB/5Oh8quMiIqMiiCC/cnIi1XERkVERRJDfvmk9E+NjJx2bGB9j+6b1FZ2RiEg9BDHw2hlcVXeNiMjJMgd5M/sH4CPAceA2d/90dHwncBVwAviou+/L+lq9bN4wpaAuIrJEpiBvZm8BrgRe6+7Pmtk50fELgC3AhcBK4A4zO9/dTyQ/m4iI5C1rTf7DwIy7Pwvg7k9Hx68EbnT3Z939MeBR4OKMryUiIillDfLnA39nZveY2X+a2Ruj41PAE12PezI6JiIiJepbrjGzO4BXxNx1bfT7K4A3AW8EbjazvwIs5vGe8PzbgG0Aq1evHuysRURkIH2DvLu/Lek+M/swcIu7O3CvmT0PnEU7cz+v66GrgCMJz78b2A3QarViPwhERGQ41o7PQ/6y2dXASnf/nJmdD9wJrAYuAP6Fdh1+ZXR8Xb+BVzN7Bvjl0CeUzVnAbyp67aromkeDrjl8f+nuZ8fdkbWF8lvAt8zsf4DngK1RVn/IzG4GHqLdWnnNIJ01SSdZBjObdfdWVa9fBV3zaNA1j7ZMQd7dnwM+kHDfdcB1WZ5fRESyCWJZAxERiacg/4LdVZ9ABXTNo0HXPMIyDbyKiEi9KZMXEQmYgryISMAU5CNm9ikzczM7q+vYTjN71MwOm9mmKs8vT2a2y8x+Zmb/bWb/bmaTXfcFec0AZnZZdF2PmtmOqs8nb2Z2npn9yMweNrNDZvax6PiZZna7mT0S/bui6nPNm5mNmdlBM/tedDv4ax6UgjztPw7gEuDxrmPdK2leBnzdzMbin6Fxbgde4+6vBX4O7ISwrzm6jq8Bb6c9We/90fWG5DjwSXd/Ne2lRq6JrnEHcKe7r6M9MTG4DzjgY8DDXbdH4ZoHoiDf9mXg05y8vk6wK2m6+w/d/Xh0827ay05AwNdM+zoedfdfRPM7bqR9vcFw96fc/b7o5z/RDnpTtK/z+uhh1wObKznBgpjZKuBy4Btdh4O+5jRGPsib2RXAnLs/sOSuUVlJ80PAD6KfQ77mkK/tFGa2BtgA3AO83N2fgvYHAXBOhadWhK/QTtKe7zoW+jUPLIjt//rps5LmZ4FL434t5lhj+k17XbO73xo95lraX/Fv6PxazOMbc819hHxtJzGzlwDfBT7u7n80i7v0MJjZO4Gn3f2Amb254tOppZEI8kkraZrZRcBa4IHoD2EVcJ+ZXUyKlTTrqNfqoQBmthV4J/BWf2GyRKOvuY+Qr22RmY3TDvA3uPst0eFfm9m57v6UmZ0LPJ38DI0zDVxhZu8AXgS81My+TdjXnMpIl2vc/UF3P8fd17j7GtqB4A3u/itgL7DFzJaZ2VpgHXBvhaebGzO7DPgMcIW7H+u6K9hrBn4KrDOztWZ2Bu0B5r0Vn1OurJ2pfBN42N2/1HXXXmBr9PNW4Nayz60o7r7T3VdFf79bgP3u/gECvua0RiKTH4a7D7WSZkN8FVgG3B59g7nb3a8O+Zrd/biZfQTYB4wB33L3QxWfVt6mgQ8CD5rZ/dGxzwIztDf0uYp2B9l7qzm9Uo3iNcfSsgYiIgEb6XKNiEjoFORFRAKmIC8iEjAFeRGRgCnIi4gETEFeRCRgCvIiIgH7f/65G01ogQKrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X_demean[:, 0], X_demean[:, 1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient Ascent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(w, X):\n",
    "    return np.sum((X.dot(w) ** 2)) / len(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def df(w, X):\n",
    "    return X.T.dot(X.dot(w)) * 2. / len(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def df_debug(w, X, epsilon = 0.0001):\n",
    "    res = np.empty(len(w))\n",
    "    for i in range(len(w)):\n",
    "        w_1 = w.copy()\n",
    "        w_1[i] += epsilon\n",
    "        w_2 = w.copy()\n",
    "        w_2[i] -= epsilon\n",
    "        res[i] = (f(w_1, X) - f(w_2, X))/(2 * epsilon)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def direction(w):\n",
    "    return w / np.linalg.norm(w)\n",
    "    # Normalize the w unit vector\n",
    "\n",
    "def first_component(df, X, initial_w, eta, n_iters = 1e4, epsilon = 1e-8):\n",
    "    \n",
    "    w = direction(initial_w)\n",
    "    i_iter = 0\n",
    "\n",
    "    while i_iter < n_iters:\n",
    "        gradient = df(w, X)\n",
    "        last_w = w\n",
    "        w = w + eta * gradient\n",
    "        w = direction(w) \n",
    "        # Normalizition the w\n",
    "\n",
    "        if (abs(f(w, X) - f(last_w, X)) < epsilon):\n",
    "            break\n",
    "\n",
    "        i_iter += 1\n",
    "\n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "initial_w = np.random.random(X.shape[1]) \n",
    "# The W can't be zero vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "eta = 0.001 # No need to standardlize X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.78025668, 0.62545944])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w = first_component(df_debug, X_demean, initial_w, eta)\n",
    "w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD4CAYAAAAJmJb0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAhb0lEQVR4nO3de5QdZZX38e+maUIHhA5ySzqEZCSE4aJGW1DDzAgZCHLNq7AG1ujKoDMRRAUGAgl4nwF6jALjIGoQEF9REgmEeMEIBGdekVtCCAEyAQS5dAIEIeAiLekk+/2jzgknnTr3qjpVdX6ftVjpU+dSVb3ofZ7az679mLsjIiL5tF2rD0BEROKjIC8ikmMK8iIiOaYgLyKSYwryIiI5tn2rD6DU7rvv7mPHjm31YYiIZMrSpUtfcfc9wp5LVZAfO3YsS5YsafVhiIhkipk9W+45pWtERHJMQV5EJMcU5EVEckxBXkQkxxTkRURyLFXVNSIiebNgWT+zF61i9boBRnV3MWPKBKZO7Els/wryIiIxWbCsn1m3rGBgcBMA/esGmHXLCoDEAr2CvIhITGYvWrUlwBcNDG5i9qJVW4J83CN9BXkRkZisXjdQcXsSI31NvIqIxGRUd1fF7ZVG+lFRkBcRicmMKRPo6uzYaltXZwczpkwAqo/0o6AgLyISk6kTe7jsY4fQ092FAT3dXVz2sUO2pGKqjfSjoJy8iEiIqCZEp07sKfu+GVMmbJWTh61H+lFQkBeRlmh1/XglUU2IVjvH4s9x/h7M3SP7sGb19va6Wg2L5N/QIArBCLY0ldFKk/oW0x+SF+/p7uKemUfW9BlJnqOZLXX33rDnlJMXkcQlUVXSjCgmRNNyjkrXiEjikqgqacao7q7QkXw9E6LlzqV/3QCT+hYnlqbSSF5EEpdEVcmCZf1M6lvMuJm/ZFLfYhYs66/5vdVKH2tR7lyMINA7b+f66zm2einIi0jiogiilRTz4Y0G02qlj7UIO0cDhs6Cxp3CUbpGRBIXd1VJLT1jajnGZo4n7BzDUkAQb5pKQV5EWqLZIFpJLT1jkijfHHqO5ap2okxTDaV0jYjkTqWcf7OpnGbEnaYKoyAvIrlTKZi2qrSxePUwMLiJDjOgsVx/vRTkRSR3Kk2ctqJ8s/TqAWCT+5Yvnbhv/lJOXkRyqVzOP4oa+HpFMRHcKI3kRaSttCIv3sqbvyIL8mbWYWbLzOwXhce7mdkdZvZk4d8RUe1LRKRRUdTA1yuJm7/KiTJdczawEtil8HgmcJe795nZzMLjCyPcn4hIQ+Is3wyTREvhciIZyZvZaOA44Aclm08Cbij8fAMwNYp9iYhkTSuuHoqiGslfCVwAvKNk217uvgbA3deY2Z5hbzSz6cB0gDFjxkR0OCIi6ZL01UNR00HezI4HXnb3pWb2kXrf7+5zgDkQ9JNv9nhERLIgqbtuoxjJTwJONLNjgR2BXczsx8BLZjayMIofCbwcwb5ERGqW1tWnolp5qhZN5+TdfZa7j3b3scCpwGJ3/wSwEJhWeNk04LZm9yUi6dFMK98ktLJ9QTVJ3nUbZ518H3CUmT0JHFV4LCI5kOYAWpSWlZnCJFk3H2mQd/ffuvvxhZ//5O6T3X184d9Xo9yXiLROmgNoUZpXn0qybl5tDUSkbmkMoEPz77t2dbJuYHCb1yVxA1I1SdbNK8iLSN1a0f+lkrCJzM4Oo3M7Y3Dz20V7Sd2AVE3ci6aUUpAXkbq18g7OMGHpo8FNzojhnQzfYfvUVddAcnXzCvIiUrckR6K1KJcmWrd+kGVfPhp4O51z7tyHW368SVKQF5GGtOoOzjDV0kdJ1qWnjVoNi0jmVWsfnIVqoLhoJC8imVctfZTGaqCkKMiLSC5USh+lrRooSUrXiEhLJdEeoRWrQaWFRvIi0jKVJkShjuqdN9+ERx+Fww4LfTpt1UBJMvf0dPft7e31JUuWtPowRCQhk/oWh6ZRRgzv5C+Dm7epw99moY3XX4erroIrrgAzeP552HHHJA49Vcxsqbv3hj2ndI2INKzZVEu5ic/X1g9WroZ55RX40pdg333hi1+ED34QFi5sywBfjdI1ItKQKGrPy02IlrPhhdUwYwZ897tBiubjH4eLL4aJE+s/gTahkbyINCSK2vNyE6LdXZ1bbRv1xst87Y7v8rvvfwouvxymTg1y8DffrABfhUbyItKQKGrPy02IAsy6ZQV7vvw8Z953Mx9/9C4cY/VJpzD2G1+H/fZr/gTahIK8iJRVafm8qGrPQ+vbH3uM9y/9PqN+vZCN23Vw22En8I4vzmLKsYc2fC7tSukaEQlVbfWnWGrPH3ooyLMffDD7/M8ddJx/HsNeeI6Tf3+rAnyDFORFJFS1nPvUiT1c9rFD6OnuwoCe7q5tSxxr9fvfw7HHwvvfD3fdFVTOPPssfOMbsPfeEZxN+1K6RkRC1ZJzb6oTpTvcfTf8+78H/+6+O1x6KXz2s7Drro19ZoIqpbLSREFeRELF1u/FHX71K7jkErj3Xhg5MqiYmT4ddtqpuc9OSJZaFytdIyKhIs+5b94M8+cHKZnjj4fVq4N696efhnPPzUyAh2y1LtZIXkRCRdbvZeNGuOmmIBWzciWMHw/XXw//+I/Q2Vn9/SmUpdbFCvIiUlZTOfcNG+BHP4LLLgtG6wcfDD/9KZxyCnR0VH9/imWpdbHSNSISrYGBoGnYu94F//IvsNtusGABLF8Op54aa4BPom0xZKt1sUbyIhKNP/8Zvvc9+Na34KWX4PDD4Qc/gKOPDjpExizJydAstS5WkBeR5qxbB//1X3DllfDqq3DUUUFnyL/920QPo9JkaBzBN00LmVeiIC+Scqmtx167NujjftVVwSj+hBOCjpBlFu6IW5YmQ5OkIC+SYqmsx169Gr75Tfj+94P8+ymnwEUXwXve05rjKcjSZGiSmp54NbN9zOxuM1tpZo+Z2dmF7buZ2R1m9mTh3xHNH65Ie0lVPfYf/whnngnjxsG3vw0nnwyPPw5z57Y8wEO2JkOTFEV1zUbgPHf/a+CDwFlmdiAwE7jL3ccDdxUei0gdUpGCeOIJOP30oL79uuuCn594Am64AQ44ILnjqCLSXjo50nS6xt3XAGsKP//ZzFYCPcBJwEcKL7sB+C1wYbP7E2kHxTx8uRWYE0lBrFgRtB6YNy9YVu+ss+D882H06Pj33aCsTIYmKdKcvJmNBSYC9wN7Fb4AcPc1ZrZnmfdMB6YDjBkzJsrDEcmkoXn4oaJKQZSd0H3wwSC433Yb7LwzXHhh0HZgz9A/4UxJ7SR2jCIL8ma2MzAfOMfd37Aa62LdfQ4wB6C3t7fcwEWkbYTl4Yt6IgpMYRO6N1/5Uz705C/Y697/ZsMuu/KjydO46qBj2GnXPZnRP8jUjMf4VE5iJyCSIG9mnQQB/kZ3v6Ww+SUzG1kYxY8EXo5iXyJ5Vy7fbsA9M4+MZB9bvkjcOfyPD/P5e+dy2POP8urO3Tz2hVlMG/Z+XtluRwDW1RkM0zpaTrqOPi2aDvIWDNmvBVa6++UlTy0EpgF9hX9va3ZfIu0giVLA1a+tZ/IfHuDzv5/Le9c8wZqd38lXJ09n7nuOZrc9RvDKkP3XGgzTPFpOxSR2C0Qxkp8EfBJYYWYPF7ZdRBDc55nZp4HngFMi2JdI7s2YMmGbnHxkpYCbNsH8+fzmR7MY/+LTPLfrXsya8jnmHzyZDdt30tPd1VQwTPNouV3r6KOorvkdwZVkmMnNfr5Iu4mlL8rgYNAB8tJLYdUq9h67HxeeeD43T/gbNm0X1JYXv0hmL1rVcDBM82g51i/PFNMdryIpFFkp4FtvwQ9/CP/xH/DMM/Dud8O8ebzjYx/jQ4+8yO/KfJHUGwxTUfJZRZaaikVJQV6kgrROIla1fj1ccw3Mng39/UE/mW9/G447bktHyHJfJPUGw6RKPqPQjnX0CvIiZaR5ErGsN94IltT71reCBmJ/93fBSH7y5Lra/dYTDJMo+YxCZr+wm6QgL1JGmicRt/Hqq8FI/T//M2j9e8wxQUfIww+PfddJlHw2K5Nf2BHRylAiZaR5EnGLl1+GmTNh333ha1+Dj3wkuGP19tsTCfBQPt+ehjx8UaoavSVMQV6kjFQHrxdegLPPhrFjg7z78cfDI4/ArbdCb2+ih5KF7o+Z+MKOidI1ImWksuTu6aeDSpnrrwd3+OQng5H8/vtv89Ioc9CVPisLVSvtWiMPCvLSJhoJeKkKXitXwmWXwU9+EiyE/c//DBdcEIzkQ9Sag67l91LLZ6W9aiWVX9gJMff09ATr7e31JUuWtPowJGfCSvy6Ojuy0Wt8+fKgI+TNN0NXF5xxBpx3HowaVfFtk/oWh45ce7q7tkyG1vp7qeWzsiDP1TVmttTdQ/N0GslL7mWqSqbo/vuD4P7zn8Muu8CsWXDOObDHHjW9vZYcdK2/l7zks9N+tREXBXnJvbQHqS0jzNfWc+xrT/KVR25lz/v/Hxt27eaHR53OVQdO4R3v2IMZL2xgam0xvqYcdK2/l3bOZ+eBgrzkXpqD1IJl/cya/wgfeOJBrrh3Loe+8DhrdxrB7dP+lS/udTh/sh0AeKPOuu5actC1/l7aOZ+dByqhlNxLbYnf5s3cd/m1zL32C/zoZ1+h5/W1fOmoMzj8Mz/gcyMnbwnwRfXUddey3mmtvxetnZptmniVtpCqSbdNm+BnPwty7o8+yh+7R3L1B0/h1oOPYLCjs+JbDXim77ittjVzbqn6vUjDKk28KsiLJGVwEH7846AU8skn4cAD+eohU/m/Yw7b0u63qMOMTSF/m0MrWjJdOSSRqRTkla4Ridtf/hI0DRs/Hj71qWBx7PnzYcUK3nvhZ9lh2NZpma7ODk47bJ+aUintfLu+1EYTryJxefNNmDMnaDuwZg186ENw9dXw0Y9u1e4Xwm+46t13t6qplLRXDknrKciLRO311+E734ErroBXXoEjj4Qbbwyah4W0+63U171ayiXNlUOSDkrXSKwWLOtnUt9ixs38JZP6FrNgWX+rDyk+f/oTfPnLQUfIiy+GQw+Fe+6Bu+6CI46oq597rVJbOSSpoZG8xKZteni/+GKwSMd3vxukaD7+cbjoInjf+2Lfdar660gqKchLbDLZTqAezz0X5NuvuSaonDnttKD9wEEHJXoY7Xq7vtRGQV5ik/ZJwYZrxP/wB+jrgxtuCB5PmwYXXgj77RfvAYs0QEFeYpPmScGGUkmPPw6XXgo//Sl0dsJnPgMzZsCYMUkdtkjdNPEqsQmbFAR4862NLZ+Arau+fNkyOPnkIA2zYAGcdx63//xeJvX8H8ZdvSL/E8qSaQryEptiz5MRw7e+VX/dwCCzblnR0sBYUyrp3nvhuOOCCdQ774QvfQmefZYFp53Nv/7PS/SvG8B5+ypAgV7SSEFeYjV1Yg/Dd9g2Kxj1XZn1lmqWXb911x3h7rth8mT48IfhgQeCHjPPPgtf/zq8851lrwLOm7dcgV5SRzl5iV3UE7BDJ0yPOGAP5i/tryu/vk37XHemPPcQlz6+EC5aCiNHwuWXw/TpsNNONR33Jvd8lohKpinIS+yinIANmzC98b7nGNrKq1qpZnH7N29fycEP3s25D8xjwuqnghuZrr4aTj8ddtyxrvMp7ve8ecu32odIK8WerjGzY8xslZk9ZWYz496fpE+1uzLrSbWEpUrK9VGteKWwcSNTH/8tv7vxbL634FIm7GRw/fVBd8gzzywb4MudT6niiF6pG0mDWIO8mXUA3wE+ChwInGZmB8a5T0mfSotOFEfmtU5i1pPiCb1S2LABrr0WDjgAPvEJ2G67oCRy5Ur4p38KSiNrPJ9KXQrUCVLSIu50zaHAU+7+NICZ3QScBDwe834lZcrdlVnvXbHlUiXG1iP60P4t11wD//Zv8Pzz0NsblEOecEIQ6BuwvRmDFdZj6F83wKS+xXXdbKVFPCRqcadreoDnSx6/UNi2hZlNN7MlZrZk7dq1MR+OpE29k7JhqRIDPvyu3aovT/fww0HO/de/DqpmTjqp4QA/e9EqBjdXXnDHoK4yy3qvakRqEfdIPuyCdqu/DHefA8yBYGWomI9HUqbeSdmpE3tY8uyrW022OvDQc69XXw3piitghx3KP1+HammjoVcWsHUKJ2y0nvteP9IScY/kXwD2KXk8Glgd8z4lQxpplXv3/66tGEDLiijAQ+XKoJ7urrKTwcXRedhoPe29fiSb4g7yDwLjzWycme0AnAosjHmfkiGVJmXLaSYYRtXfPuzLqXM7Y8TwTlavG6CjzKxsh1nZ0XrZG7RS0OtHsivWdI27bzSzzwGLgA7gOnd/LM59SjpVmlCst1Vuo3X3Ufa3H9rHfdeuTt7csJHX1g8ChC7C3dXZsU2AL1q9boAr/uG9oYtyawEQaUbsdfLu/it339/d3+Xul8S9P0mfqCcUG10NKepFr6dO7OGemUfyTN9x7DRsewY3bRvYO8y2ukLpqTBab+SqRqQa3fEqsasWXOstGWx0NaQ4c97lPmOzO8/0HQcEX3ZvvrVxm9eUfkFpARCJmoK8xK5cACyO6BtJnzQSDOPsb1/ts4emiopGDO/kKyccpMAusVEXSolduSBaaRIyDnEuel3ts8OuZgCG77C9ArzESkFeYlcuAIZNTkJ8JYNx5ryLn93d9XZbhB073/7zUnmktIrSNRK7cjn02YtWJb48YNw577c2bt7y82vrB7ekn9K8FKLkm4K8xCKsZPKemUdu87o8lQxWmmDepn892T5XyQ6layRytZZMDk2fjBjeybDtt+PcuQ9nct3USikZlUdKq2gkL5GrpwdLMX0S5Y1KrVItJaPySGkFjeQlco1MMkZ9o1IrxFm9I9IojeRTKOs9xRuZZGzkiyFtv6dGb9ISiZOC/BCtDhx5SFs0MslY7xdDWn9PSslI2ihdUyINizbkIW3RyCRjvamOPPyeRJKgkXyJNCzakJebZuod0dab6sjL76nVV46SfwryJdIQONr5ppl6vhjy8HtKa8pJ8kXpmhJpWLRBFRq1ycPvSSknSYJG8iUamTCM+nI7bxUacaUj8vB7SsOVo+SfgnyJegNHXJfbYWmLLOZu405HZL2SJQ8pJ0k/Bfkh6gkcSU3UZjV3m4aJ7DRTPxtJgnLyTUjqcjuruVulIypTPxtJgkbyTUjqcjuKYLlgWT9fXfgY6waChaaTWJFI6Yjqsp5ykvTTSH6IBcv6mdS3mHEzf1m1E2JSFR7NVv0sWNbPjJ8t3xLgIeh1PuPm5bHe6JWHChiRrFOQL1HvHa9JXW43GyxnL1rF4OZtV2Ea3OSxpnzymo6oZyAg0mpK15RoZKKw0cvteqplmi0XrJTWiTs/nrd0RFYnwaV9KciXiHqisFwgbyRQNBMsy+XGi89J7VQxJFmjIF+i2kRhPaPvSoE86UAxY8oEZvxs+TYpm84Oiz0/nsX6/kpUMSRZo5x8iUq573rz9ZUCedKBYurEHmaf8h66uzq3bBsxvJPZJ78n1oCbhq6eUUtD6wuRemgkX6JS7ntS3+LQoH3O3Ic5Z+7D25QkVgrkrSgtjCo3Xs/IPI+pDd3AJFmjID9EuWBYbZRdLEksfkalQB51oEgqJVLvXEIeUxt56Jkj7aWpIG9ms4ETgA3AH4DT3X1d4blZwKeBTcAX3H1Rc4dam7gCXqXJy6JiSeLUiT0VA3mUgSLJao96R+Z5vRkqbxVDkm/N5uTvAA5293cDTwCzAMzsQOBU4CDgGOBqM+so+ykRiTMHHJavD1McpVarEZ86sYd7Zh7JM33Hcc/MIxsOGkm2PKh3ZK6boURar6mRvLv/puThfcDJhZ9PAm5y97eAZ8zsKeBQ4N5m9ldNnDng0tF3pRF96Sg1iRFfkimRekfmSm2ItF6UOflPAXMLP/cQBP2iFwrbtmFm04HpAGPGjGnqAKLq8VIuKBWDdrFNQCtKEocqF3gdmNS3eKvjbzaV1chcglIbIq1VNcib2Z3A3iFPXezutxVeczGwEbix+LaQ1297Xz3g7nOAOQC9vb2hr6lVszngWvPbxZ+TbvgVJizwFpUeP9B07l4j89bL230HEj9zbyquYmbTgDOAye6+vrBtFoC7X1Z4vAj4qrtXTNf09vb6kiVLGj6WoUEagpFmrf1SJvUtDv2S6Onu4p6ZRzZ8XNU0+4dbfH+5NFJP4UuuFecm0Wn2/2/JLzNb6u69Yc81NfFqZscAFwInFgN8wULgVDMbZmbjgPHAA83sqxbNNsRqRclfFJPFxUncsMsnCI4/j+WM7Sar6wpIazWbk78KGAbcYWYA97n7Ge7+mJnNAx4nSOOc5e7b5hNiEEePlzhL/qKcLK52/HksZ2wn+qKWRjQ1knf3/dx9H3d/b+G/M0qeu8Td3+XuE9z99uYPNX6NlPw123Y2yj/cSsevcsbsU0sFaYTueC3RioW8o7x6qOX4NWmXXWqpII1oeuI1Ss1OvCYtiolaTaZJPVRdI2EqTbxqJN+EKFItKkuUeui+A6mXgnxBIyOkqFIt+sMVkbionzyNlzFqMlNE0k4jeRovY8xaqkX5XJH2k4sgX++yfENf20xuPSuplqRaEuuLRCRdMp+uqSfVUu613cM7t3kt5Kv+OIm7JfO43J9I1mV+JF8teJWOKtdv2Bj62mHbb0dXZ0eu64+TuFsyj8v9iWRd5kfy5YJUcRRZOqp8bf1g6GtfHxhsqudNFiRxt6RuuxdJn8yP5LuHd5YN3mHtd8OM6u7KTG69UUncLZnX5f5EsizzI/lmb9jt6uzgiAP2aKr/TBY026GzFiopFUmfzI/kXx8IH8WX093VyU7Dtt+Spz/igD2Yv7Q/tqqTNFWbxH21krWSUpF2kPkgXy5FEKars4Ovnrj16k2T+hbHNlmYVNlimuQ97SWSNZlP14SlCEp1mFVMT8Q5WahFHkSk1TI/kg9bb7Wolm6OcU4WqtpERFot8yN5CAL9w185miv/4b11TyzGOVmoRR5EpNUyP5Iv1Ug+OM7JQi3yICKtlqsg36i4JgtVbSIiraYgHzNVm4hIK+UiJy8iIuFyMZJP0w1HaT4mEWk/mQ/yrbjhqFoAb8eboEQknTKfrqnnhqMFy/qb7lFTS8903QQlImmR+SBf6w1HUS1oUUsA101QIpIWmQ/ytd5wFNXoupYArpugRCQtMh/ka71jtVpwrjWVU0sAV8tdEUmLzAf5WvukVwrO9aRyagngSfRuFxGphXmzq24AZnY+MBvYw91fKWybBXwa2AR8wd0XVfuc3t5eX7JkSdPHE2ZoxQu83cBs9qJVoU3Kerq7uGfmkaGfpfJIEUkLM1vq7r1hzzVdQmlm+wBHAc+VbDsQOBU4CBgF3Glm+7t7bevxxaBSi4Fz5z4c+p5yKR7dxSoiWRFFnfwVwAXAbSXbTgJucve3gGfM7CngUODeCPbXsHLBWWuTikheNZWTN7MTgX53Xz7kqR7g+ZLHLxS2hX3GdDNbYmZL1q5d28zhNEwTpSKSV1VH8mZ2J7B3yFMXAxcBR4e9LWRbaPLf3ecAcyDIyVc7njioW6SI5FXVIO/ufx+23cwOAcYBy80MYDTwkJkdSjBy36fk5aOB1U0fbYyUZxeRPGo4XePuK9x9T3cf6+5jCQL7+9z9RWAhcKqZDTOzccB44IFIjlhERGoWS4Myd3/MzOYBjwMbgbNaWVkjItKuIgvyhdF86eNLgEui+nwREalf5u94FRGR8hTkRURyTEFeRCTHFORFRHJMQV5EJMcU5EVEckxBXkQkxxTkRURyTEFeRCTHFORFRHJMQV5EJMcU5EVEckxBXkQkxxTkRURyTEFeRCTHFORFRHJMQV5EJMcU5EVEckxBXkQkxxTkRURyLLKFvFttwbJ+Zi9axep1A4zq7mLGlAlMndjT6sMSEWmpXAT5Bcv6mXXLCgYGNwHQv26AWbesAFCgF5G2lot0zexFq7YE+KKBwU3MXrSqRUckIpIOuQjyq9cN1LVdRKRd5CLIj+ruqmu7iEi7yEWQnzFlAl2dHVtt6+rsYMaUCS06IhGRdMjFxGtxclXVNSIiW2s6yJvZ54HPARuBX7r7BYXts4BPA5uAL7j7omb3VcnUiT0K6iIiQzQV5M3sCOAk4N3u/paZ7VnYfiBwKnAQMAq408z2d/dN5T9NRESi1mxO/kygz93fAnD3lwvbTwJucve33P0Z4Cng0Cb3JSIidWo2yO8P/I2Z3W9m/21mHyhs7wGeL3ndC4VtIiKSoKrpGjO7E9g75KmLC+8fAXwQ+AAwz8z+CrCQ13uZz58OTAcYM2ZMbUctIiI1qRrk3f3vyz1nZmcCt7i7Aw+Y2WZgd4KR+z4lLx0NrC7z+XOAOQC9vb2hXwQiItIYC+Jzg282OwMY5e5fNrP9gbuAMcCBwE8I8vCjCtvHV5t4NbO1wLMNH1BzdgdeadG+W0Xn3B50zvm3r7vvEfZEsyWU1wHXmdmjwAZgWmFU/5iZzQMeJyitPKuWyppyB5kEM1vi7r2t2n8r6Jzbg865vTUV5N19A/CJMs9dAlzSzOeLiEhzctHWQEREwinIv21Oqw+gBXTO7UHn3MaamngVEZF000heRCTHFORFRHJMQb7AzM43Mzez3Uu2zTKzp8xslZlNaeXxRcnMZpvZ/5rZI2Z2q5l1lzyXy3MGMLNjCuf1lJnNbPXxRM3M9jGzu81spZk9ZmZnF7bvZmZ3mNmThX9HtPpYo2ZmHWa2zMx+UXic+3OulYI8wR8HcBTwXMm20k6axwBXm1lH+Cdkzh3Awe7+buAJYBbk+5wL5/Ed4KMEN+udVjjfPNkInOfuf03QauSswjnOBO5y9/EENybm7gsOOBtYWfK4Hc65JgrygSuAC9i6v05uO2m6+2/cfWPh4X0EbScgx+dMcB5PufvThfs7biI439xw9zXu/lDh5z8TBL0egvO8ofCyG4CpLTnAmJjZaOA44Aclm3N9zvVo+yBvZicC/e6+fMhT7dJJ81PA7YWf83zOeT63bZjZWGAicD+wl7uvgeCLANizhYcWhysJBmmbS7bl/Zxrlovl/6qp0knzIuDosLeFbMtMvWmlc3b32wqvuZjgEv/G4ttCXp+Zc64iz+e2FTPbGZgPnOPub5iFnXo+mNnxwMvuvtTMPtLiw0mltgjy5TppmtkhwDhgeeEPYTTwkJkdSh2dNNOoUvdQADObBhwPTPa3b5bI9DlXkedz28LMOgkC/I3ufkth80tmNtLd15jZSODl8p+QOZOAE83sWGBHYBcz+zH5Pue6tHW6xt1XuPue7j7W3ccSBIL3ufuLwELgVDMbZmbjgPHAAy083MiY2THAhcCJ7r6+5KncnjPwIDDezMaZ2Q4EE8wLW3xMkbJgpHItsNLdLy95aiEwrfDzNOC2pI8tLu4+y91HF/5+TwUWu/snyPE516stRvKNcPeGOmlmxFXAMOCOwhXMfe5+Rp7P2d03mtnngEVAB3Cduz/W4sOK2iTgk8AKM3u4sO0ioI9gQZ9PE1SQndKaw0tUO55zKLU1EBHJsbZO14iI5J2CvIhIjinIi4jkmIK8iEiOKciLiOSYgryISI4pyIuI5Nj/ByRAryatdo1oAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X_demean[:, 0], X_demean[:, 1])\n",
    "plt.plot([0, w[0] * 50], [0, w[1] * 50], color = 'red')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "X2 = np.empty(( 100, 2))\n",
    "X2[:, 0] = np.random.uniform(0., 100., size = 100)\n",
    "X2[:, 1] = 0.75 * X2[:, 0] + 3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X2_demean = demean(X2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2 = first_component(df, X_demean, initial_w, eta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.78025668, 0.62545944])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD8CAYAAACSCdTiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAgn0lEQVR4nO3de3xU1bn/8c9DDBrwEiioGEixHGoVFdFU9MQqRSnUCyCKhaIiXlBLvVREQTiitir+csDa1tbilSpeOIIDtVUExBtHLmLQAIKCFyCgeAHrkRRC8vz+mBlMwkwyMDOZycz3/Xrxmpm998xe29Yvy7Wftba5OyIikpmapboBIiKSPAp5EZEMppAXEclgCnkRkQymkBcRyWAKeRGRDJawkDezHDMrNbPnQ59bm9kcM/sg9NoqUecSEZHYJLInfx3wXo3Po4F57t4ZmBf6LCIijSghIW9m7YGzgIdqbO4HTAm9nwL0T8S5REQkdvsk6Hd+D9wEHFBj2yHuvgnA3TeZ2cEN/UibNm28Y8eOCWqSiEh2WLp06Rfu3jbSvrhD3szOBja7+1Iz67EX3x8ODAcoLCzkrbfeirdJIiJZxcw+ibYvEcM1xUBfM/sYeBroaWZPAJ+ZWbtQA9oBmyN92d0nu3uRuxe1bRvxLyIREdlLcYe8u49x9/bu3hEYBLzs7hcCs4ChocOGAjPjPZeIiOyZZNbJTwB6mdkHQK/QZxERaUSJuvEKgLu/ArwSev8lcHoif19ERPaMZryKiGSwhPbkRURkzwRKyymZvZqNWys4LD+PUb2PoH+3goT9vkJeRCRFAqXljJlRRkVlFQDlWysYM6MMIGFBr+EaEZEUKZm9elfAh1VUVlEye3XCzqGevIhIIxny4JssWPtVg8dt3FqRsHOqJy8i0ghiDXiAw/LzEnZehbyISCOINeDzcnMY1fuIhJ1XIS8ikmIF+XlY6PXuAceoukZEJN3VLY2sz4LRPZPWDoW8iEiCRSqNbGZQ7bsfW9ypdVLbouEaEZEEi1QaWe2w7z61I7e4U2umXnFyUtuinryISJzqDs2URymB3LGzmo8nnNWobVPIi4jEIdLQjAERRmYSWhoZKw3XiIjEIdLQjANW57hEl0bGSiEvIhKHaLNTneSWRsZKwzUiIjGKtGJktDH4gvy8pJZGxkohLyJSj0BpObf/fQVbtlXW2h5eMfK8EwqYvrS81pBNqoZmItFwjYhIFIHSckY9+85uAR9WUVnF/FWfc/eAY9JiaCYS9eRFRJYvh2uvhb/9Ddq337W5ZPZqKqsi1cl8Z+PWCvp3K0ibUK8r7pA3s/2A14B9Q7/3rLuPN7PWwDNAR+Bj4AJ33xLv+UREEqa6Gu69F265BfLz4ZNPCHxuu8bd64/3oFSURe6JRAzXbAd6untX4Digj5mdBIwG5rl7Z2Be6LOISHr45BM4/XS48UY480xYvpxAi46MmVFGeYwBn05j79HE3ZN3dwf+L/QxN/THgX5Aj9D2KcArwM3xnk9EJC7u8PjjcM01wfePPEKgay9KHi6LOlM1klYtchl/Tpe0HaYJS8iYvJnlAEuB/wDud/dFZnaIu28CcPdNZnZwIs4lIrLXvvgCrroKpk+Hn/wEpkwhsLV5rRmrDcnPy+W2vukf7mEJCXl3rwKOM7N84DkzOzrW75rZcGA4QGFhYSKaIyKyuxdegEsvhS+/hHvuIdBzECXPrImp954uNe97I6EllO6+leCwTB/gMzNrBxB63RzlO5Pdvcjdi9q2bZvI5oiIwLffwtVXB8fd27aFJUsI9BrCmJkrYwr4pjDuXp+4Q97M2oZ68JhZHnAGsAqYBQwNHTYUmBnvuURE9siiRdCtG/z1r8EbrIsXE6huw8hp78Q0PJNuNe97IxHDNe2AKaFx+WbANHd/3szeBKaZ2WXAOmBgAs4lItKwykr47W/hrrugoADmz4fTTtu1YmSV1187k5eb0+TDPSwR1TXvAt0ibP8SOD3e3xcR2SOrVsGFF8LSpTB0KNx3Hxx0EBB5xci6CkJr0mRCwINmvIpIpqiuhvvvh5tugpYtYfp0Aod3p+QvSxt8mAdkVu+9JoW8iDR95eUwbBjMmcOnp/RkSPcrWLt4X1i87LtD6nmYR45ZRgY8KORFpKl75plg9cz27Sy7ZQIDq4+mMsqQe/hhHjV3Z2oPPkyrUIpI07RlC/zylzBoEBxxBCxbxogDfhw14MPS5WEejUU9eRFpeubOhUsugc8+Y+WvRnHVYb1Y//D7Ma0305QnNu0N9eRFpOmoqIDrroNeveCAA5j/2CzOa92Tdd/syJgFxRJNPXkRaRqWLg2WRq5axdrBl3L5kefzUVl1zF9vKguKJZpCXkTS286dcM89cNttVLRuw28uvpsX2x0D38YW8Nka7mEKeRFJX2vWwEUXwcKFrO/Tj/5HDubL5vs3+LVsG3evj0JeRNKPO0yeDDfcAM2bw1NP0ff91lGftVpTNo6710c3XkUkvXz6KZx9dnDd9+JiKCuDQYNiCvhsKIncU+rJi0j6mDEDhg8PLg/8hz/AiBHQrOG+aKZPaIqHevIiknpffx2sez/vPOjYEUpLg4/nqxHw+Xm5Eb9qoICvh0JeRFLr1Veha1d4/HFWXXEdp557F4c/tpbiCS8TKC3fddhtfbuQ28xqfTW3mXHvL45TwNdDIS8iqbF9O4waBT/9KeTm8uojz3HuIX1Y900lTnBBsTEzynYFff9uBZQM7FprSYKSgV0V8A0wb2Dx/MZUVFTkb731VqqbISLJ9u67wYlNZWU81/0cxv7nULbvmxfxYR4qh2yYmS1196JI+3TjVUQaT1UVTJoE48bx7wMO4rpBtzP7+ycE90XpcG6M4TmsEp1CXkQax8cfw8UXw+uvw4ABDDjqQlZWNm/wa4fl5yW/bRksEQ/y7mBm883sPTNbYWbXhba3NrM5ZvZB6LVV/M0VkSbHHR57DI49FpYtgylT4NlneS+GgNfEpvgl4sbrTmCkux8JnASMMLOjgNHAPHfvDMwLfRaRbPL55zBgAAwbxtttfsApQ+6jeGN7Ass2Ru2h55hlzVrvjSERD/LeBGwKvf/GzN4DCoB+QI/QYVOAV4Cb4z2fiDQRzz8Pl11G1Zat/PcZl/PA8X1xawahqpnzTihg+tLyWg/W1qSmxEtoCaWZdQS6AYuAQ0J/AYT/Ijg4kecSkTT1f/8HV14J55wDhx7KJb+6n7+c0D8Y8CEVlVXMX/U5dw84Jque0pQKCbvxamb7A9OB6939X2bW0FfC3xsODAcoLCxMVHNEJBX+93/h4oup/vBDJp90PpOKh7Bjn8gzVTduraB/twKFepIlpCdvZrkEA36qu88Ibf7MzNqF9rcDNkf6rrtPdvcidy9q27ZtIpojIo1txw4YOxZ+8hO2fFPBLwbfzYTTLoka8KCqmcYSd0/egl32h4H33H1SjV2zgKHAhNDrzHjPJSJpaOXK4MSm0lK49FJ6HPRzvm5ef4CraqbxJKInXwxcBPQ0s2WhP2cSDPdeZvYB0Cv0WUQyRXU13HcfHH88bNgAgQA8/HC9Aa+x98aXiOqaNwj+bxfJ6fH+voikofXrYdgwmDcvuPb7Qw/BIYcAwRLISMsT5Jix9u4zG7ulWU8LlIlI7NzhySepPOpotr2+gNF9rqH4P68nsHHnrkMGd+8Q8avRtktyaVkDEYnNV1/B1VfDtGm8U3AkN5x1A+tatYOv/82YGWVAcKXI3/U/BoCnFq2nyp0cMwZ377BruzQurUIpIg2bPZuKi4ayz5dfcO8pQ3ig+3lUN8updYhWi0yd+lah1HCNiES3bVvwCU19+lDu+9L/4kn8+eQLdgt40GqR6UrDNSIS2ZIlcNFFsHo1Txefx/iThrB9n+iLiqnuPT2pJy8ite3cCXfcASefHHyg9ty5jDllWL0Bb6C69zSlnryIfOf994O998WLefG4M7jptMs5YIlxUF4uWysqI37FgCEnFaruPU0p5EUkWBr5wAMwciQ7cptz84AxPNe5GIB/ba0gN8fIbWZUVtcu1MjPy+W2vl0U8GlMIS+S5V546W3yr7mKk99fwsLOP+bWvr/h/X0OrHVMZZXTqkUuLZrvw8atFRyWn8eo3kco3JsAhbxIFltc8ldOHj+KfSt3MK7X1TzR7UyIsoLs1m2VlN76s0ZuocRLIS+Sjb7+Gq65hhMff5xl7Tpzw1kj+fB77ev9iqpnmiaFvEi2mT8fhg6FjRu5t/iX3H/yBezMqT8KtGpk06WQF8kSsxauZduNNzNowXTWtWnPh48GeLa8BTsjTGLKz8ul5b4af88ECnmRLDD/6Zc48trhdP78E6YcfxZ39xiGvb8P553QNuJzVlUxkzk0GUokgwXeWsd9Z1xG8ZCzOLDiG4YOvJ3xva7m37n76TmrWUI9eZEMNfGBFzj1dzfSv3wlzx9xCuN6/4qtebVLI/Wc1cynkBfJNO48d80dXPngPVRbM647eyQzj+oRsTRSFTOZTyEvkkk++4xNF1zEua/NYcH3j+XGM3/DpgPbRjxUFTPZQSEvkilmzYLLL6f1lq3c0fMKHi06B7fIt91yzDT2niUScuPVzB4xs81mtrzGttZmNsfMPgi9tkrEuUSkjm++gcsvh379oKCAcy7+PY/8uF/UgM/NMSZe0FUBnyUSVV3zGNCnzrbRwDx37wzMC30WkUR64w3o2hUefRTGjIFFi/i284+iHt6yeQ4l5yvgs0lCQt7dXwO+qrO5HzAl9H4K0D8R5xIRYMeOYKifemrwhuprr8Fdd0Hz5ozqfQR5ubWf3GTAhScVsuKOPgr4LJPMMflD3H0TgLtvMrODk3gukeyxfDlceCG88w5ccQVMnAgHHLBrdzjES2av1oxVSf2NVzMbDgwHKCwsTHFrRNJXYOl61t16F1fOfphv81ry/r2PctL1l0Q8VrXvEpbMGa+fmVk7gNDr5kgHuftkdy9y96K2bSOXeolks0BpOX1+8wQHDziba//5AK/+4AR6XfInhn15KIHS8lQ3T9JcMnvys4ChwITQ68wknkskIwXe3sCbt9/HtBf/TDOcUT+/lv85pldwHL6yipLZq9Vjl3olJOTN7CmgB9DGzDYA4wmG+zQzuwxYBwxMxLlEssaXX3Lg0CHcs/w1Frc/ihvOuoEN+YfWOmRjhBUkRWpKSMi7++Aou05PxO+LZJ0XX4Rhwzhl8xfc3eMSHvzxuVQ3y9ntMC1LIA1J+Y1XEanh229h1Cj4y1/g6KO54oI7eDXvsIiHalkCiYWWGhZJF4sWQbdu8MADMHIkLFnCuZecuVvNO0CrFrlalkBiop68SAoFSsuZ9M8VnPfPxxix8Bl2HNyOFi+/DD16AKp5l/gp5EVSIFBazm2zVvC9DR/yx+cn0fXTD5h+dE8m9PkVYw/qXGt6uGreJR4KeZFGFigtZ8z0dxm4aCa3vPIo23L346r+Y3jxiGIAlUVKQinkRRrZo9Pe4K9P38OpH5cy/wcncNPPr+Pz/Vvv2q+ySEkkhbxIY3rmGab8/nKaV1Uy9me/YupxP9/tiU0qi5REUsiLJNG4QBlPLVpPy4pv+O2cB+i38hXKOxzJiD7X83Hr3YdkVBYpiaaQF0mScYEynli4juKPl/Hf/7iXNtu2MvGUISwbciWflX8LlVW1jm/VIpfx53TReLwklEJeJElmvLGGW195jEuXzmJN6/YMHzCOsnadyfnkGyZe0FVlkdIoFPIiCRIoLef2v69gy7ZKuny6hpnPT6Tzl+t59IRzuOe0ofw7dz8AqtxVFimNRiEvkgDhoZmc6ipGLHyW6xc8yZctDuLCC37LG4d3q3VsTp0brSLJpJAXiVOgtJypC9fx/S0bmfT8JE7YuIpZR57Kf/W6mq/zDtjt+MHdO6SglZKtFPIicSp5cRWDlr3IuJcfYmezHK49ZxSzjjptt+NyzBjcvQO/639MClop2UohLxKPTz/ljofHcPraJbz+/eMYdeb1fHpgm1qHFOTnsWB0zxQ1ULKdQl5kDwVKyymZvZqjF89jwkv3U7y9gvFnXMnfjj8Lt9oLu+Y2M9W9S0op5EViULNyZv/t2xg/dzIDl8+l7JBOjBpyI2u/V4hXe63v5OU24+4Bx6qKRlJKIS/SgCEPvsmCtV8BcOL65Ux6fhLtvvmCP5z8C/5YPIjKnFxa7bcPLZrvo7p3STsKeZEoAqXljJnxLhWV1TTfWckNrz/O8MXP8UmrQxk45B7eLjhy17Fbt1VSeuvPUthakciSHvJm1ge4D8gBHnL3Cck+p0i8ggFfRkVlNT/a/BH3Pj+RIz//mKnH9eHOn17Gtua1FxHTomKSrpIa8maWA9wP9AI2AEvMbJa7r0zmeUXiVTJ7Ndu372D4kgAjX3+cf+23P8POH8/8Tj/e7VgtKibpLNk9+ROBNe7+IYCZPQ30AxTyktbsk4956h/30n39cl784cnc0vvXfNXioN2O06Jiku6SHfIFwPoanzcA3WseYGbDgeEAhYWFSW6OSAzc+cs/J9Lxs48YeeZvmH50z93WfM9tBiUDj1O4S9pLdshHWqSjVp2Zu08GJgMUFRV5hONFGpcZmyf+ketfXc/alrUnNjUz+GX3Qs1alSYj2SG/Aai5UEd7YGOSzykSt9PP78k3ncq1HLA0eckO+SVAZzM7HCgHBgG/TPI5RaIKz1aNJbi1HLBkgqSGvLvvNLNfA7MJllA+4u4rknlOkWjGBcqYunDdrvHC8q0VjJlRBqAwl4zVrOFD4uPu/3T3H7p7J3e/M9nnE4kkvBxw3Zs+FZVVlMxenZI2iTQGzXiVjFVzaKaZ2W4BH7Zxa0WjtkukMSnkJSN9N2M1+LDsKo9euKXZqpLJkj5cI5IKJbNX7wr4+hhotqpkNIW8ZKRYhmAMGHJSoW66SkbTcI1kpMPy8yiPEPQ5ZlS7q+5dsoZCXjLSqN5H1BqTh+BCYncPOEbBLllFIS9NUqC0nLHPlfHtjmCIh4dewssNhINcM1Yl2ynkpckJlJYz8n/eoarG4/YceGLhOoBaQa9Ql2ynG6/S5JTMXl0r4Gt6atH6iNtFspVCXpqc+ipn6quHF8lGGq6RtBZpQbFolTMQrJ4Rke+oJy9pKzxrtXxrBc53C4r99EdtyWkWOcwHd+8QcbtItlLIS1oKlJYzcto7u81araisYv6qz5k4sCstm+fs2m7AhSfpYR4idWm4RtJOuAcfbXx949YKVc6IxEg9eUk7Da07owXFRGKnnrykVLDX/i4VldVA8BmqUaojgeCsVS0oJhI7hbykzJAH32TB2q9qbasv4HPMtCyByB7ScI2kxLhA2W4BX5+83BwmXtBVAS+yh+IKeTMbaGYrzKzazIrq7BtjZmvMbLWZ9Y6vmZIpAqXlFE94edcSBPUpyM/DQq/qwYvsnXiHa5YDA4C/1txoZkcBg4AuwGHAXDP7obs3/BQHyUiB0nJu//sKtmyrjOn4HDMWjO6Z5FaJZL64Qt7d3wOw3WcZ9gOedvftwEdmtgY4EXgznvNJ0xMoLee2WSvYWhFbuIdpUpNIYiRrTL4AqLlS1IbQNski4Xr3PQ344k6tNalJJEEa7Mmb2Vzg0Ai7xrr7zGhfi7AtYt2EmQ0HhgMUFhY21BxpIsIzVvdkwbCWzXO481yNvYskUoMh7+5n7MXvbgBq/vd2e2BjlN+fDEwGKCoq0hKCGaChGas16WlNIsmVrDr5WcCTZjaJ4I3XzsDiJJ1L0kik2vdo8vNyua1vFwW8SBLFFfJmdi7wR6At8A8zW+buvd19hZlNA1YCO4ERqqzJfLEGfKsWuYw/R+Eu0hjira55Dnguyr47gTvj+X1pWhoK+BwzTWgSaWSa8SqNQjNWRVJDa9fIHon0pKZYgls3V0VSQyEvMRsXKGPqwnW7amHDT2oC6N+tgOJOrSMO2RR3aq2AF0kRDddITAKl5bUCPqyisoqS2asBmHrFyRR3al1rf3Gn1ky94uRGaqWI1KWevMSkZPbqyLPZCD6pKUyBLpJe1JOXmNQM8rr0pCaR9KWQl5hEC3IDPalJJI0p5KWW8Hrvh4/+B8UTXiZQWg4EgzwvN6fWsQYMOalQN1VF0pjG5GWX8Joz4Ydo162eAfaqfFJEUkchL7tq38sjjLuHq2f6dyvY9UdEmg6FfJarW/seSX03XUUkvWlMPotFq32vS9UzIk2XQj6L1Vf7HpaXm6PqGZEmTMM1WayhYZgC3VwVafIU8lnssPy8iDdbDbj3F8cp3EUygIZrsphq30Uyn3ryGSjW5YBV+y6S+RTyGSaWCU01qfZdJLNpuCbDlMxevSvgw2ouBywi2SXeB3mXAOcAO4C1wDB33xraNwa4DKgCrnX32fE1VaIJlJZz+99XsGVbZdRjNKFJJDvF25OfAxzt7scC7wNjAMzsKGAQ0AXoA/zZzHKi/orstXGBMq5/Zlm9AQ+a0CSSreLqybv7SzU+LgTOD73vBzzt7tuBj8xsDXAi8GY855PvxNJ7D9OEJpHslcgx+UuBF0LvC4D1NfZtCG2TBAjfXI0l4Avy8/QQbZEs1mBP3szmAodG2DXW3WeGjhkL7ASmhr8W4fiIM+jNbDgwHKCwsDCGJkukm6uRFOTnsWB0z0ZokYikqwZD3t3PqG+/mQ0FzgZOd/dwkG8AOtQ4rD2wMcrvTwYmAxQVFTW0lErWqln7Hss/pNxmpiEaEYlvuMbM+gA3A33dfVuNXbOAQWa2r5kdDnQGFsdzrmwWHp4pjzHgW+Q2o2RgVw3RiEjck6H+BOwLzDEzgIXufpW7rzCzacBKgsM4I9y94fEFiSjW4Zn8vFxu69tF4S4iu8RbXfMf9ey7E7gznt+XoPpq3A20HIGIRKVlDZqAaKtF6saqiDREyxo0AZFWi1Ttu4jEQj35JkCrRYrI3lLIp1CsSwKDVosUkb2jkE+RPV0SWERkb2hMPkW0JLCINAaFfIpEK4vUksAikkgarkmyaOPu0coitSSwiCSSevJJVHc5gvC4e6C0XGWRItIoFPJJVN+4e/9uBdw94BgK8vMwtCSwiCSHhmuSqKFxd5VFikiyqSefRNHG1zXuLiKNRSGfRBp3F5FU03BNnOqbtarlCEQk1RTycYhl1qrG3UUklRTye6hmz72ZGVVe+1lNNatnRERSTSG/B3pNeoUPNn+763PdgA/TrFURSRe68RqjugFfH1XPiEi6UE++HuMCZUxdtI4oHfaIVD0jIukkrp68mf3WzN41s2Vm9pKZHVZj3xgzW2Nmq82sd/xNbVxDHnyTJxbGFvA5Zpq1KiJpKd6efIm7/xeAmV0L3ApcZWZHAYOALsBhwFwz+6G7V0X/qfQRKC1nwdqvYj5+4gVdFewikpbi6sm7+79qfGwJhPu9/YCn3X27u38ErAFOjOdcjWlP1nTvfHBLBbyIpK24x+TN7E7gYuBr4KehzQXAwhqHbQhti/T94cBwgMLCwnibkxCxVsd0Prglc27okdzGiIjEocGQN7O5wKERdo1195nuPhYYa2ZjgF8D4wGLcHzE0W13nwxMBigqKtqDW5yJMS5QxlOL1lPlTo4Zg7t3iLrWe1h+Xi639e2iHryIpL0GQ97dz4jxt54E/kEw5DcAHWrsaw9s3OPWJdm4QBlPLFy363OVO08sXEdxp9Z89e2O3ZYJbp5j/L/zNf4uIk1HXMM1ZtbZ3T8IfewLrAq9nwU8aWaTCN547QwsjudciRSetRqtt77wwy1MvKCr1pwRkSYv3jH5CWZ2BFANfAJcBeDuK8xsGrAS2AmMSJfKmrrrzURS5a41Z0QkI8QV8u5+Xj377gTujOf3E6Wh9WbqyrFItxRERJqejJ/xWrfn3lDAAwzu3qHBY0REmoKMD/lIz1mNJlxd87v+xyS5VSIijSMjQ77m8EwsNZl5uTlajkBEMlLGhXwsN1Yh2GuvdlfljIhktIwL+ViGZ9RzF5FskREhH+vwjIF67iKSVZp8yMc6PFOQn8eC0T0bqVUiIumhyT8ZKtbhGT3IQ0SyUZPvyde3YqSGZ0Qk2zX5kI+2YqSGZ0REMmC4ZlTvI8jLzam1TcMzIiJBTb4nHx6G0YqRIiK7a/IhD2jFSBGRKJr8cI2IiESnkBcRyWAKeRGRDKaQFxHJYAp5EZEMZh7Dk5Iai5l9TvBZsemiDfBFqhuRQrp+Xb+uv2n4vru3jbQjrUI+3ZjZW+5elOp2pIquX9ev62/616/hGhGRDKaQFxHJYAr5+k1OdQNSTNef3XT9GUBj8iIiGUw9eRGRDKaQr4eZ3WhmbmZtamwbY2ZrzGy1mfVOZfuSxcxKzGyVmb1rZs+ZWX6Nfdlw/X1C17fGzEanuj3JZmYdzGy+mb1nZivM7LrQ9tZmNsfMPgi9tkp1W5PJzHLMrNTMng99zojrV8hHYWYdgF7AuhrbjgIGAV2APsCfzSwn8i80aXOAo939WOB9YAxkx/WHrud+4OfAUcDg0HVnsp3ASHc/EjgJGBG65tHAPHfvDMwLfc5k1wHv1ficEdevkI/uXuAmoOZNi37A0+6+3d0/AtYAJ6aiccnk7i+5+87Qx4VA+9D7bLj+E4E17v6hu+8AniZ43RnL3Te5+9uh998QDLoCgtc9JXTYFKB/ShrYCMysPXAW8FCNzRlx/Qr5CMysL1Du7u/U2VUArK/xeUNoWya7FHgh9D4brj8brjEqM+sIdAMWAYe4+yYI/kUAHJzCpiXb7wl26qprbMuI68+Ih4bsDTObCxwaYddY4BbgZ5G+FmFbkyxPqu/63X1m6JixBP9Tfmr4axGOb5LXX49suMaIzGx/YDpwvbv/yyzSP4rMY2ZnA5vdfamZ9UhxcxIua0Pe3c+ItN3MjgEOB94J/Z+8PfC2mZ1IsFfXocbh7YGNSW5qUkS7/jAzGwqcDZzu39XZZsz11yMbrnE3ZpZLMOCnuvuM0ObPzKydu28ys3bA5tS1MKmKgb5mdiawH3CgmT1Bhly/hmvqcPcydz/Y3Tu6e0eC/9If7+6fArOAQWa2r5kdDnQGFqewuUlhZn2Am4G+7r6txq5suP4lQGczO9zMmhO80TwrxW1KKgv2Zh4G3nP3STV2zQKGht4PBWY2dtsag7uPcff2oX/fBwEvu/uFZMj1Z21Pfm+4+wozmwasJDiMMcLdq1LcrGT4E7AvMCf0XzML3f2qbLh+d99pZr8GZgM5wCPuviLFzUq2YuAioMzMloW23QJMAKaZ2WUEq8wGpqZ5KZMR168ZryIiGUzDNSIiGUwhLyKSwRTyIiIZTCEvIpLBFPIiIhlMIS8iksEU8iIiGUwhLyKSwf4/gh1GJq8KiuwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X2_demean[:, 0], X2_demean[:, 1])\n",
    "plt.plot([0, w2[0] * 50], [0, w2[1] * 50], color = 'red')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The n-ed Principal Component"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def direction(w):\n",
    "    return w / np.linalg.norm(w)\n",
    "    # Normalize the w unit vector\n",
    "\n",
    "def first_component(X, initial_w, eta, n_iters = 1e4, epsilon = 1e-8):\n",
    "    \n",
    "    w = direction(initial_w)\n",
    "    i_iter = 0\n",
    "\n",
    "    while i_iter < n_iters:\n",
    "        gradient = df(w, X)\n",
    "        last_w = w\n",
    "        w = w + eta * gradient\n",
    "        w = direction(w) \n",
    "        # Normalizition the w\n",
    "\n",
    "        if (abs(f(w, X) - f(last_w, X)) < epsilon):\n",
    "            break\n",
    "\n",
    "        i_iter += 1\n",
    "\n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.78025615, 0.6254601 ])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "initial_w = np.random.random(X.shape[1]) \n",
    "eta = 0.01\n",
    "w = first_component(X_demean, initial_w, eta)\n",
    "w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_2 = X_demean - X_demean.dot(w).reshape(-1, 1) * w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD4CAYAAAAJmJb0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWwklEQVR4nO3df2zc9X3H8dc7xiCDKhlECuSIG6CpJZiLvVokKFrVrhTTdisHUhYyMjG1alqJTgpF1pKSUaBhRLU6kLZ1a5BQkZKFQJuYtGkJv6ShVTXUkR2cwFzCj/ywUTFLvErLCYz93h93l17sO9+d7773ve/3ng/J8t33+737fi6Xe+WTz72/n4+5uwAA8bQo7AYAAIJDyANAjBHyABBjhDwAxBghDwAxdk7YDch18cUX+7Jly8JuBgBEyoEDB95398X59tVVyC9btkyDg4NhNwMAIsXMjhbax3ANAMQYIQ8AMUbIA0CMEfIAEGOEPADEWF1V1yxU/9CY+vaPanwypSWtLertaVeyKxF2swAgdJHvyfcPjWnT7hGNTabkksYmU7pr17A294+E3TQACF3kQ75v/6hSU9NnbXNJOwaOqX9oLJxGAUCdiHzIj0+m8m53pf8BAIBGFvmQX9LaUnBfoX8AAKBRRD7ke3vaZQX2zfcPAAA0gsiHfLIrodtXts0J+pbmJvX2tIfSJgCoF5EPeUnakuzQw2s6lWhtkUlKtLbooVs7KKME0PBiUScvpXv0hDoAnC0WPXkAQH6EPADEGCEPADFGyANAjBHyABBjhDwAxBghDwAxVpWQN7PHzOw9MzuUs+0iM3vOzN7I/L6wGucCAJSuWj35n0i6ada2jZJecPflkl7I3AcA1FBVQt7dX5J0ctbmmyU9nrn9uKRkNc4FAChdkNMaXOLu70qSu79rZh/Pd5CZrZe0XpLa2toCbE71sNwggKgI/YtXd9/m7t3u3r148eKwm1NUvuUGN+waVuf9z7ISFYC6E2TI/97MLpOkzO/3AjxXzeRbblCSJlNT2rBrWFdu2sf6sgDqRpAhv1fSHZnbd0h6OsBz1Uyx1aZmXNo+cIygB1AXqlVCuVPSbyS1m9kJM/u6pK2Svmhmb0j6YuZ+5JW62tR2FhIHUAeq8sWru68tsOsL1Xj+etLb065Nu0fyDtnMtml3ujfPl7IAwhL6F69Rk+xK6KFbO3Th+c1Fj01NTatv/2gNWgUA+RHyC5DsSmjo3hv1yJpOtTTP/0dYbAwfAIJEyFcg2ZXQ69//ktatLFzfX+oYPgAEgZCvgi3Jjkyvvums7S3NTertaQ+pVQAQo4W8w5b9crWUK2G5YhZArZi7h92GM7q7u31wcDDsZgQqe8VsbnWOSbp9ZZu2JDvCaxiAyDKzA+7enW8fwzU1lu+KWZe0g7p6AAEg5GusULWNS5RbAqg6xuRrbElri8YKBP3YZEqrtr7IWD2AqqEnX2O9Pe2yAvtMOmt2y7t2DTMHDoCKEPI1luxK6PaVbXOC3pQessnlSs+Bs2zjPnU9wFTGAMpHyIdgS7JDD6/pVKK1RSYp0doyJ+BnO3V6Snc/dZCgB1AWSijrxKqtLxYcq8914fnNGrr3xhq0CEBUUEIZAfON1ec6dXoq8LYAiA+qa+pEsiuhwaMntWPgWNGhG66YBVAqevJ1JDtWP980xuc3L5qzxuym3SOM1QPIizH5OrW5f0TbB46dta15kemC887RZCr/kI2ZdPsKpkcAGg1j8hGUndkytwKnb/W1+t8CAS9Jnllf9vZHf1O7hgKoa/TkI6bUKhwmPQMaBz35GOntaZ8zb30+2QupuGIWaGyBh7yZvWNmI2Y2bGZ00yuUXWO2yUopuJR2vnw84BYBqGe16sl/3t07C/13AuVJdiX0w7+6VotKyPlpd63a+iLVN0CDCnxM3szekdTt7u8XO5Yx+fL0D42p96lhTc2U/pgmM61dsZSxeiBGwh6Td0nPmtkBM1s/e6eZrTezQTMbnJiYqEFz4iPZldAb//gVPbKmUxecW3ycXkr37BmrBxpHLXryS9x93Mw+Luk5SX/n7i/lO5aefGU2949o58vHNV3Ge7rqqou04xvXB9gqAEELtSfv7uOZ3+9J2iPpuqDP2ai2JDv05kNfVqK1peTH/PrNk/rUPb9kzB6IqUBD3swuMLOPZW9LulHSoSDPidLLLLM+nHamRgBiKuie/CWS/svMDkp6RdI+d38m4HM2vGyZZTk9+tTUtDbsGqYSB4gZrnhtAPnmwZkPV8sC0RJ2dQ1Clp0Hp7TLp9LlUDsGjtGjB2KAkG8Qya6EHl7TqXObSot6lxi+AWKA4ZoGlV14hMnOgOibb7iGkG9w/UNjumvXcNHVqLLWEfZA3WFMHgUluxK6fWVbyeP12weOafl39zGEA0QEIY8zyw6WWnI5NSP1PnWQoAcigOEanKXc4ZsEC4kDoWO4BiXLDt+UamwypQ27hpnwDKhThDzm2JLs0Loygl5Kj9UzfAPUH0IeeWUvoCq1rl6S+vaPBtgiAAvBmDyK6h8a0/0/P6xTp6eKHptobdH4ZEpLGKsHaoY6eVTNlZv2aabAXxmTzvrClouogNrgi1dUzV+vKDxWPzv7Xemx+mUb96nrgWcZswdCQMijLNkvZS1nqP785uJ/jU6dntJ3nhwm6IEaY7gGVbFq64slzYMjSY+s6WSsHqgihmsQuN6e9pKnRtiwa5jhG6BGCHlURblz4Jw6PcWSg0ANEPKomuwcOBee31zS8Sw5CASPkEdVJbsSGrr3xrKumB2bTOkupkYAAkHIIxBbkh1addVFJR+fLbck6IHqCjzkzewmMxs1syNmtjHo86F+7PjG9Vq3sk2LSp8Z4Uxdfef9fDELVEOgJZRm1iTpd5K+KOmEpN9KWuvur+U7nhLKeCtnycEsVqICiguzhPI6SUfc/S13/1DSE5JuDvicqFPJroR+vfHP9ciazpKrcHYwuyVQkaBDPiHpeM79E5ltZ5jZejMbNLPBiYmJgJuDelDOnPUuZrcEKhF0yOfrsJ01PuTu29y92927Fy9eHHBzUC/KmbN+vIzhHQBnCzrkT0hamnP/cknjAZ8TEVFq0C8pce1ZAHMFHfK/lbTczK4ws3Ml3SZpb8DnRIRkFycpdAFVS3OTenvaa9wqID7OCfLJ3f0jM/u2pP2SmiQ95u6HgzwnoifZlTgzYVm2AqfQwiP9Q2PatPtVpaZmJEmLLD39MRU4QH7MQonI6B8a03d2DWsmz76W5kV66NZPM7slGhKzUCIW+vaP5g14SUpNzTDhGZAHIY/IKFZlk5qaptwSmIWQR2SUUmUzNpnSFRv3MbMlkEHIIzJ6e9pL+gvrSof9hl3DuubeZwh7NDRCHpGR7Eron9Z0qqWENWWz/u/D9Jz1V//Drwh7NKRASyiBaitUblmsRuz01IzufurgmecAGgU9eURWdsKzt7d+RYkSxuunZ1z3/5zLNNBYCHnEQqkLiZ86PRV4W4B6QsgjFsqZ2RJoJIzJIza2JDvU/YmL9N3dr+r0VP7Lplpb0nPkbO4f0c6Xj2vaXU1mWrtiKVMjIJaY1gCxtLl/RNsHjp21rXmRqW/1tRo8enLOPompERBdTGuAhpOd3TLR2iKTlGhtUd/qa5XsSmjny8fzPiY1NaMNu4Z1+6O/qW1jgQAxXIPYyi23zDVd5H+vv37zpDb3jzB8g1igJ4+G02TF63C2s7YsYoKQR8NZu2Jp8YMk3bVrWMuYBwcRR8ij4WSXHVxUpEOfHdTJzoPTef+zhD0ih5BHQ9qS7NBbD31Fq666qOTHTKamdPdTBwl6RAohj4a24xvXl7SYeNb0jGvDrmGGcBAZhDwaXrbcsqW5qeTHjE2mWIkKkUDIA0qXWz50a0dJE51lpaam6dWj7gUW8mZ2n5mNmdlw5ufLQZ0LqIbsrJaPrOlUc7FvZXPQq0c9C7on/7C7d2Z+fhnwuYCqSHYl1Lf6Wl1wbunDN9lefdcDVOCgvjBcA+SR7Ero8AM3ad3KtpIunso6dXpKG3YNa3P/SICtA0oX2ARlZnafpL+V9AdJg5LudvdTeY5bL2m9JLW1tX3m6NGjgbQHqFR2JaqxyVRJx69b2cbUCKiJ+SYoqyjkzex5SZfm2XWPpAFJ7yt9Tcn3JV3m7l+b7/mYhRJR0D80pk27R5Samp73OJP08JpOZrVE4OYL+YomKHP3G0pswKOSflHJuYB6kQ3tu588OO9kZy6pb/8oIY9QBTYLpZld5u7vZu7eIulQUOcCai0b3L1PHdTUTOGgH5tMqfP+ZzWZSi87eOH5zfreX15D8KNmgpxq+Adm1ql0h+YdSd8M8FxAzWWDer6VqCSdCXgp/cVs708PnvV4IEiBVde4+9+4e4e7f9rdv5rTqwdiI9mV0Gvf/5LWrWybs5B4oZqcqWlX3/7RoJsGSKKEEqiKLckOPTxrJar5ShrGS6zQASrFylBAlcxeiWrV1hcLllsuKWP6BKAShDwQkN6e9rxfzDY3mXp72rW5f0Q7Xz6uaXc1mWntiqXU1aPqCHkgINle/X17D8+prhk8elLbB46dOXbaXdsHjmn7wDElWlvU29POF7OoCkIeCFChxcTvfvJgwcdkJzzLPh6oBF+8AiGY7yIqKT3hGRU4qAZCHghBKZOeUYGDaiDkgRCsXbG06DFU4KAaGJMHQpCtoslW18zW0tyk3p72WjcLMRTYVMMLwSyUaFTZaYzHJ1NaUqC6ppRj0JgCm2q42gh5IL/+oTH1/vSgpqbP/rwyZz2k+UOeMXkgAu7/+eE5AS9J2weOsdwg5kXIAxFw6vRUwX2UWmI+hDwQcZRaYj6EPBABrS3NBfdRaon5EPJABNz31Wvyflizk50BhVAnD0TAfJOdUUaJ+RDyQEQUmuwMmA8hD8QYF1CBkAdiqn9oTJt2jyg1NS2JKYwbVUVfvJrZajM7bGYzZtY9a98mMztiZqNm1lNZMwGUq2//6JmAz2IK48ZTaU/+kKRbJf04d6OZXS3pNknXSFoi6Xkz+5S7T899CgBBKFQ/T119Y6moJ+/ur7t7vm7BzZKecPcP3P1tSUckXVfJuQCUp1D9PHX1jSWoOvmEpOM5909kts1hZuvNbNDMBicmJgJqDtB4enva1dLcdNY2pjBuPEWHa8zseUmX5tl1j7s/Xehhebblne7S3bdJ2ialZ6Es1h4Apcl+uUp1TWMrGvLufsMCnveEpNylby6XNL6A5wFQgXJr6ym5jJ+ghmv2SrrNzM4zsyskLZf0SkDnAlAF2ZLLscmUXH8suWQq42irtITyFjM7Iel6SfvMbL8kufthSU9Kek3SM5LupLIGqG+UXMZTRSWU7r5H0p4C+x6U9GAlzw+gdii5jCdmoQQgiZLLuCLkAUii5DKumLsGgCRKLuOKkAdwRqkll5v7R7Tz5eOadleTmdauWKotyY4atBDlIuQBlGVz/4i2Dxw7c3/a/cx9gr7+MCYPoCw7Xz5e1naEi5AHUJZpzz/7SKHtCBfDNQDK0mSWN9CbzBirr0P05AGUZe2KpXm3X7n4fG0fOHbmH4DsWP3m/pFaNg+zEPIAyrIl2aF1K9vUZOnJZpvMtG5lm96aOJ33eMbqw8VwDYCybUl2zBmGya24ycVYfbjoyQOoimzPvtTtqA1CHkBVFBqrL7QdtcFwDYCqyA7fUF1TX8zraLysu7vbBwcHw24GAESKmR1w9+58+xiuAYAYI+QBIMYIeQCIMUIeAGKMkAeAGKso5M1stZkdNrMZM+vO2b7MzFJmNpz5+ffKmwoAKFeldfKHJN0q6cd59r3p7p0VPj8AoAIVhby7vy5JxmXLAFCXghyTv8LMhszsP83szwodZGbrzWzQzAYnJiYCbA4ANJ6iPXkze17SpXl23ePuTxd42LuS2tz9f8zsM5L6zewad//D7APdfZukbVL6itfSmw4AKKZoyLv7DeU+qbt/IOmDzO0DZvampE9JYs4CAKihQIZrzGyxmTVlbl8pabmkt4I4FwCgsEpLKG8xsxOSrpe0z8z2Z3Z9VtKrZnZQ0k8lfcvdT1bWVABAuSqtrtkjaU+e7T+T9LNKnhsAUDmueAWAGCPkASDGCHkAiDFCHgBijJAHgBgj5AEgxgh5AIgxQh4AYoyQB4AYI+QBIMYIeQCIMUIeAGKMkAeAGCPkASDGCHkAiDFCHgBijJAHgBgj5AEgxgh5AIgxQh4AYqyikDezPjP7bzN71cz2mFlrzr5NZnbEzEbNrKfilgIAylZpT/45SX/i7p+W9DtJmyTJzK6WdJukayTdJOlHZtZU4bkAAGWqKOTd/Vl3/yhzd0DS5ZnbN0t6wt0/cPe3JR2RdF0l5wIAlK+aY/Jfk/SrzO2EpOM5+05kts1hZuvNbNDMBicmJqrYHADAOcUOMLPnJV2aZ9c97v505ph7JH0kaUf2YXmO93zP7+7bJG2TpO7u7rzHAAAWpmjIu/sN8+03szsk/YWkL7h7NqRPSFqac9jlksYX2kgAwMJUWl1zk6S/l/RVdz+ds2uvpNvM7Dwzu0LSckmvVHIuAED5ivbki/gXSedJes7MJGnA3b/l7ofN7ElJryk9jHOnu09XeC4AQJkqCnl3/+Q8+x6U9GAlzw8Acdc/NKa+/aMan0xpSWuLenvalezKW6eyIJX25AEAC9Q/NKZNu0eUmkoPdIxNprRp94gkVS3omdYAAELSt3/0TMBnpaam1bd/tGrnIOQBICTjk6myti8EIQ8AIVnS2lLW9oUg5AEgJL097WppPntar5bmJvX2tFftHHzxCgAhyX65SnUNAMRUsitR1VCfjeEaAIgxQh4AYoyQB4AYI+QBIMYIeQCIMfvjFPDhM7MJSUcrfJqLJb1fhebUK15fdMX5tUm8vjB9wt0X59tRVyFfDWY26O7dYbcjKLy+6Irza5N4ffWK4RoAiDFCHgBiLI4hvy3sBgSM1xddcX5tEq+vLsVuTB4A8Edx7MkDADIIeQCIsdiEvJmtNrPDZjZjZt2z9m0ysyNmNmpmPWG1sVrM7D4zGzOz4czPl8NuU6XM7KbM+3PEzDaG3Z5qM7N3zGwk834Nht2eSpnZY2b2npkdytl2kZk9Z2ZvZH5fGGYbF6rAa4vsZy42IS/pkKRbJb2Uu9HMrpZ0m6RrJN0k6Udm1jT34ZHzsLt3Zn5+GXZjKpF5P/5V0pckXS1pbeZ9i5vPZ96vyNVa5/ETpT9PuTZKesHdl0t6IXM/in6iua9NiuhnLjYh7+6vu3u+1W9vlvSEu3/g7m9LOiLputq2DkVcJ+mIu7/l7h9KekLp9w11yt1fknRy1uabJT2euf24pGQt21QtBV5bZMUm5OeRkHQ85/6JzLao+7aZvZr5r2Uk/1ucI67vUS6X9KyZHTCz9WE3JiCXuPu7kpT5/fGQ21NtkfzMRSrkzex5MzuU52e+Xp/l2Vb3daNFXuu/SbpKUqekdyX9MMy2VkEk36MyrXL3P1V6SOpOM/ts2A1CWSL7mYvU8n/ufsMCHnZC0tKc+5dLGq9Oi4JT6ms1s0cl/SLg5gQtku9ROdx9PPP7PTPbo/QQ1UvzPypyfm9ml7n7u2Z2maT3wm5Qtbj777O3o/aZi1RPfoH2SrrNzM4zsyskLZf0SshtqkjmA5R1i9JfOkfZbyUtN7MrzOxcpb8o3xtym6rGzC4ws49lb0u6UdF/z/LZK+mOzO07JD0dYluqKsqfuUj15OdjZrdI+mdJiyXtM7Nhd+9x98Nm9qSk1yR9JOlOd58Os61V8AMz61R6SOMdSd8MtTUVcvePzOzbkvZLapL0mLsfDrlZ1XSJpD1mJqU/c//h7s+E26TKmNlOSZ+TdLGZnZD0PUlbJT1pZl+XdEzS6vBauHAFXtvnovqZY1oDAIixRhiuAYCGRcgDQIwR8gAQY4Q8AMQYIQ8AMUbIA0CMEfIAEGP/DwpursQFNd6eAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X_2[:, 0], X_2[:, 1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.62545724,  0.78025845])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w_2 = first_component(X_2, initial_w, eta)\n",
    "w_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.666103585542313e-06"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w.dot(w_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def first_n_component(n, X, eta = 0.01, n_iters = 1e4, epsilon = 1e-8):\n",
    "    \n",
    "    X_pca = X.copy()\n",
    "    X_pca = demean(X_pca)\n",
    "    res = []\n",
    "    for i in range(n):\n",
    "        initial_w = np.random.random(X_pca.shape[1]) \n",
    "        w = first_component(X_pca, initial_w, eta)\n",
    "        res.append(w)\n",
    "        \n",
    "        X_pca = X_pca - X_pca.dot(w).reshape(-1, 1) * w\n",
    "    \n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([0.78025624, 0.62545999]), array([-0.62545624,  0.78025924])]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_n_component(2, X)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}