state_political_affiliation_crawl.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import re\n",
    "import pandas as pd\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_request = requests.get(\"https://en.wikipedia.org/wiki/Political_party_strength_in_U.S._states\")\n",
    "main_page = BeautifulSoup(main_request.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "states = main_page.select('.wikitable.sortable tr')[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_names = []\n",
    "house_blue_perc = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "for state in states:\n",
    "    fields = state.select('td')\n",
    "    state_name = fields[0].text.replace(\"\\n\", \"\")\n",
    "    \n",
    "    if state_name == 'Nebraska':\n",
    "        continue\n",
    "    \n",
    "    state_names.append(state_name)\n",
    "    \n",
    "    party_in_power, party_ratio = fields[3].text.replace(\"\\n\", \"\").split(' ')\n",
    "    \n",
    "    party_numbers = list(map(lambda x: int(''.join(list(filter(str.isdigit, x)))), party_ratio.split('–')))\n",
    "    \n",
    "    if party_in_power == 'Republican':\n",
    "        house_blue_perc.append(round(party_numbers[1]/sum(party_numbers)*100, 2))\n",
    "    else:\n",
    "        house_blue_perc.append(round(party_numbers[0]/sum(party_numbers)*100, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_house_df = pd.DataFrame({\n",
    "    'state': state_names,\n",
    "    'state_house_blue_perc': house_blue_perc\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_request = requests.get(\"https://en.wikipedia.org/wiki/List_of_United_States_governors\")\n",
    "main_page = BeautifulSoup(main_request.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "states = main_page.select('.wikitable.sortable')[0].select('tr')[2:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_names = []\n",
    "governor_party = []\n",
    "\n",
    "for state in states:\n",
    "    fields = state.select('td')\n",
    "    state_name = fields[0].select('a')[-1].text\n",
    "    \n",
    "    if state_name == 'Nebraska':\n",
    "        continue\n",
    "    \n",
    "    state_names.append(state_name)\n",
    "    party = fields[4].text\n",
    "    \n",
    "    if 'Repu' in party:\n",
    "        governor_party.append('red')\n",
    "    else:\n",
    "        governor_party.append('blue')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "governor_df = pd.DataFrame({\n",
    "    'state': state_names,\n",
    "    'state_governor_party': governor_party\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(\n",
    "    state_house_df,\n",
    "    governor_df,\n",
    "    left_on='state',\n",
    "    right_on='state'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>state</th>\n",
       "      <th>state_house_blue_perc</th>\n",
       "      <th>state_governor_party</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Alabama</td>\n",
       "      <td>22.86</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Alaska</td>\n",
       "      <td>35.00</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Arizona</td>\n",
       "      <td>43.33</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Arkansas</td>\n",
       "      <td>25.71</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>California</td>\n",
       "      <td>72.50</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Colorado</td>\n",
       "      <td>54.29</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Connecticut</td>\n",
       "      <td>61.11</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Delaware</td>\n",
       "      <td>57.14</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Florida</td>\n",
       "      <td>42.50</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Georgia</td>\n",
       "      <td>37.50</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Hawaii</td>\n",
       "      <td>96.00</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Idaho</td>\n",
       "      <td>20.00</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Illinois</td>\n",
       "      <td>67.80</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Indiana</td>\n",
       "      <td>20.00</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Iowa</td>\n",
       "      <td>36.00</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Kansas</td>\n",
       "      <td>27.50</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Kentucky</td>\n",
       "      <td>23.68</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Louisiana</td>\n",
       "      <td>30.77</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Maine</td>\n",
       "      <td>60.00</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Maryland</td>\n",
       "      <td>68.09</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Massachusetts</td>\n",
       "      <td>85.00</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Michigan</td>\n",
       "      <td>42.11</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Minnesota</td>\n",
       "      <td>47.76</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Mississippi</td>\n",
       "      <td>30.77</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Missouri</td>\n",
       "      <td>29.41</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Montana</td>\n",
       "      <td>40.00</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Nevada</td>\n",
       "      <td>61.90</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>New Hampshire</td>\n",
       "      <td>58.33</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>New Jersey</td>\n",
       "      <td>62.50</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>New Mexico</td>\n",
       "      <td>61.90</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>New York</td>\n",
       "      <td>63.49</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>North Carolina</td>\n",
       "      <td>42.00</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>North Dakota</td>\n",
       "      <td>21.28</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Ohio</td>\n",
       "      <td>27.27</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Oklahoma</td>\n",
       "      <td>18.75</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Oregon</td>\n",
       "      <td>60.00</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>44.00</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Rhode Island</td>\n",
       "      <td>86.84</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>South Carolina</td>\n",
       "      <td>41.30</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>South Dakota</td>\n",
       "      <td>14.29</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Tennessee</td>\n",
       "      <td>15.15</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Texas</td>\n",
       "      <td>38.71</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Utah</td>\n",
       "      <td>20.69</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Vermont</td>\n",
       "      <td>73.33</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Virginia</td>\n",
       "      <td>52.50</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Washington</td>\n",
       "      <td>59.18</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>West Virginia</td>\n",
       "      <td>41.18</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Wisconsin</td>\n",
       "      <td>42.42</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Wyoming</td>\n",
       "      <td>10.00</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             state  state_house_blue_perc state_governor_party\n",
       "0          Alabama                  22.86                  red\n",
       "1           Alaska                  35.00                  red\n",
       "2          Arizona                  43.33                  red\n",
       "3         Arkansas                  25.71                  red\n",
       "4       California                  72.50                 blue\n",
       "5         Colorado                  54.29                 blue\n",
       "6      Connecticut                  61.11                 blue\n",
       "7         Delaware                  57.14                 blue\n",
       "8          Florida                  42.50                  red\n",
       "9          Georgia                  37.50                  red\n",
       "10          Hawaii                  96.00                 blue\n",
       "11           Idaho                  20.00                  red\n",
       "12        Illinois                  67.80                 blue\n",
       "13         Indiana                  20.00                  red\n",
       "14            Iowa                  36.00                  red\n",
       "15          Kansas                  27.50                 blue\n",
       "16        Kentucky                  23.68                 blue\n",
       "17       Louisiana                  30.77                 blue\n",
       "18           Maine                  60.00                 blue\n",
       "19        Maryland                  68.09                  red\n",
       "20   Massachusetts                  85.00                  red\n",
       "21        Michigan                  42.11                 blue\n",
       "22       Minnesota                  47.76                 blue\n",
       "23     Mississippi                  30.77                  red\n",
       "24        Missouri                  29.41                  red\n",
       "25         Montana                  40.00                 blue\n",
       "26          Nevada                  61.90                 blue\n",
       "27   New Hampshire                  58.33                  red\n",
       "28      New Jersey                  62.50                 blue\n",
       "29      New Mexico                  61.90                 blue\n",
       "30        New York                  63.49                 blue\n",
       "31  North Carolina                  42.00                 blue\n",
       "32    North Dakota                  21.28                  red\n",
       "33            Ohio                  27.27                  red\n",
       "34        Oklahoma                  18.75                  red\n",
       "35          Oregon                  60.00                 blue\n",
       "36    Pennsylvania                  44.00                 blue\n",
       "37    Rhode Island                  86.84                 blue\n",
       "38  South Carolina                  41.30                  red\n",
       "39    South Dakota                  14.29                  red\n",
       "40       Tennessee                  15.15                  red\n",
       "41           Texas                  38.71                  red\n",
       "42            Utah                  20.69                  red\n",
       "43         Vermont                  73.33                  red\n",
       "44        Virginia                  52.50                 blue\n",
       "45      Washington                  59.18                 blue\n",
       "46   West Virginia                  41.18                  red\n",
       "47       Wisconsin                  42.42                 blue\n",
       "48         Wyoming                  10.00                  red"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('./relevant_data/state_party_line.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}