RapWordsTalk.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Word! Automating a Hip-hop word of the day blog\n",
    "==============================================="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Chris Ing, @jsci http://rapwords.tumblr.com (Soon: https://github.com/cing/rapwords/)\n",
    "----------------------------------------------------------------------------\n",
    "* * *\n",
    "![Rap Words Post](assets/Advisory.png)\n",
    "* * *\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Rap Words Logo](assets/RapWordsLogo.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Rap Words Post](assets/RapWordsPost1.png)\n",
    "![Rap Words Post](assets/RapWordsPost2.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Requirements\n",
    "============\n",
    "\n",
    "- standard library (re, glob, collections, html)\n",
    "- pandas (http://pandas.pydata.org/), numpy\n",
    "- wiktionaryparser (https://github.com/Suyash458/WiktionaryParser)\n",
    "- spotipy (https://github.com/plamere/spotipy) / Google Data API (https://github.com/google/google-api-python-client)\n",
    "- nltk (https://github.com/nltk/nltk)\n",
    "- pypronouncing (https://github.com/aparrish/pronouncingpy) \n",
    "- pytumblr (Python3 fork) (https://github.com/jabbalaci/pytumblr) / oauthlib / oauthlib_requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "import re\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading Lyrics into Memory\n",
    "=========================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from html.parser import HTMLParser\n",
    "\n",
    "class MLStripper(HTMLParser):\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "        self.strict = False\n",
    "        self.convert_charrefs= True\n",
    "        self.fed = []\n",
    "    def handle_data(self, d):\n",
    "        self.fed.append(d)\n",
    "    def get_data(self):\n",
    "        return ''.join(self.fed)\n",
    "\n",
    "def strip_tags(html):\n",
    "    s = MLStripper()\n",
    "    s.feed(html)\n",
    "    return s.get_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_data = defaultdict(list)\n",
    "for filename in glob.iglob('Lyrics/ohhla.com/*/*/*/*.txt', recursive=True):\n",
    "    with open(filename, 'r', encoding = \"ISO-8859-1\") as f:\n",
    "        stripped_lyrics = strip_tags(f.read())\n",
    "        \n",
    "        artist = re.search('Artist:\\s*(.*)\\s*\\n', stripped_lyrics)\n",
    "        song = re.search('Song:\\s*(.*)\\s*\\n', stripped_lyrics)\n",
    "        lyrics = re.search('Typed by:\\s*(.*)\\s*\\n([\\s\\S]*)', stripped_lyrics)\n",
    "\n",
    "        if artist is not None and song is not None and lyrics is not None:\n",
    "            df_data[\"filename\"].append(filename)\n",
    "            df_data[\"artist\"].append(artist.group(1))\n",
    "            df_data[\"song\"].append(song.group(1))\n",
    "            df_data[\"lyrics\"].append(lyrics.group(2).lower())  # group(1) is the transcriber"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist</th>\n",
       "      <th>filename</th>\n",
       "      <th>lyrics</th>\n",
       "      <th>song</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>2 Chainz f/ The Weeknd</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_chainz/TRUstory/l...</td>\n",
       "      <td>[chorus: the weeknd]\\ngirl, i'm just another b...</td>\n",
       "      <td>Like Me</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>2 Chainz</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_chainz/TRUstory/m...</td>\n",
       "      <td>[intro: 2 chainz]\\n(yeah~!) i told them get on...</td>\n",
       "      <td>Money Machine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>2 Chainz f/ Drake</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_chainz/TRUstory/n...</td>\n",
       "      <td>[intro]\\n{*mike will made it*}\\nyo.. t.r.u.! (...</td>\n",
       "      <td>No Lie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>2 Chainz f/ Dolla Boy</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_chainz/TRUstory/s...</td>\n",
       "      <td>[intro: r&amp;b samples]\\nnothing in the whole wid...</td>\n",
       "      <td>Stop Me Now</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>2 Chainz f/ Lil Wayne</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_chainz/TRUstory/y...</td>\n",
       "      <td>[intro]\\nyuck daddy! yuck!\\nyuck daddy! yuck!\\...</td>\n",
       "      <td>Yuck!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>2 Hungry Bros. present 8thW1 f/ Janelle Renee</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_hungry/no_room/mo...</td>\n",
       "      <td>[intro]\\ngoo-oh-oh-ohhh {*2x*}\\n(c'mon people!...</td>\n",
       "      <td>More Go</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>2 Hungry Bros. present 8thW1</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_hungry/no_room/sh...</td>\n",
       "      <td>[intro/chorus - interpolation of krs-one]\\nthe...</td>\n",
       "      <td>Short and Sweet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>2 Hungry Bros. present 8thW1</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_hungry/no_room/sy...</td>\n",
       "      <td>[8thw1]\\nthe teacher, class in session\\nthe be...</td>\n",
       "      <td>Say My Name Right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>2 Hungry Bros. present 8thW1</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_hungry/no_room/ta...</td>\n",
       "      <td>[8thw1]\\nwhy stress mic skills? i upgrade to l...</td>\n",
       "      <td>Talkin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>2 Live Crew</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_live/2liveis/chec...</td>\n",
       "      <td>[mr. mixx]\\ncheck it out y'all\\nch-ch-check it...</td>\n",
       "      <td>Check it Out Y'all</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>2 Live Crew</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_live/2liveis/get_...</td>\n",
       "      <td>( *mr. mixx cuts in dolemite* )\\n(i wouldn't l...</td>\n",
       "      <td>Get it Girl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>2 Live Crew</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_live/2liveis/thro...</td>\n",
       "      <td>[ fresh kid ice ]\\nlisten up y'all 'cause this...</td>\n",
       "      <td>Throw the 'D'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>2 Live Crew</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_live/2liveis/want...</td>\n",
       "      <td>chorus(4x):  luke\\nsomebody say hey-y-y-y-y we...</td>\n",
       "      <td>We Want Some Pussy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>2 Live Crew</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_live/2liveis/what...</td>\n",
       "      <td>1-2 1-2\\ni am the dominating fresh kid ice\\nan...</td>\n",
       "      <td>2 Live is What We Are... (Word)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>2 Live Crew</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_live/as_nasty/7_b...</td>\n",
       "      <td>[note:  brother marquis does all the verses.]\\...</td>\n",
       "      <td>My Seven Bizzos</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            artist  \\\n",
       "105                         2 Chainz f/ The Weeknd   \n",
       "106                                       2 Chainz   \n",
       "107                              2 Chainz f/ Drake   \n",
       "108                          2 Chainz f/ Dolla Boy   \n",
       "109                          2 Chainz f/ Lil Wayne   \n",
       "110  2 Hungry Bros. present 8thW1 f/ Janelle Renee   \n",
       "111                   2 Hungry Bros. present 8thW1   \n",
       "112                   2 Hungry Bros. present 8thW1   \n",
       "113                   2 Hungry Bros. present 8thW1   \n",
       "114                                    2 Live Crew   \n",
       "115                                    2 Live Crew   \n",
       "116                                    2 Live Crew   \n",
       "117                                    2 Live Crew   \n",
       "118                                    2 Live Crew   \n",
       "119                                    2 Live Crew   \n",
       "\n",
       "                                              filename  \\\n",
       "105  Lyrics/ohhla.com/anonymous/2_chainz/TRUstory/l...   \n",
       "106  Lyrics/ohhla.com/anonymous/2_chainz/TRUstory/m...   \n",
       "107  Lyrics/ohhla.com/anonymous/2_chainz/TRUstory/n...   \n",
       "108  Lyrics/ohhla.com/anonymous/2_chainz/TRUstory/s...   \n",
       "109  Lyrics/ohhla.com/anonymous/2_chainz/TRUstory/y...   \n",
       "110  Lyrics/ohhla.com/anonymous/2_hungry/no_room/mo...   \n",
       "111  Lyrics/ohhla.com/anonymous/2_hungry/no_room/sh...   \n",
       "112  Lyrics/ohhla.com/anonymous/2_hungry/no_room/sy...   \n",
       "113  Lyrics/ohhla.com/anonymous/2_hungry/no_room/ta...   \n",
       "114  Lyrics/ohhla.com/anonymous/2_live/2liveis/chec...   \n",
       "115  Lyrics/ohhla.com/anonymous/2_live/2liveis/get_...   \n",
       "116  Lyrics/ohhla.com/anonymous/2_live/2liveis/thro...   \n",
       "117  Lyrics/ohhla.com/anonymous/2_live/2liveis/want...   \n",
       "118  Lyrics/ohhla.com/anonymous/2_live/2liveis/what...   \n",
       "119  Lyrics/ohhla.com/anonymous/2_live/as_nasty/7_b...   \n",
       "\n",
       "                                                lyrics  \\\n",
       "105  [chorus: the weeknd]\\ngirl, i'm just another b...   \n",
       "106  [intro: 2 chainz]\\n(yeah~!) i told them get on...   \n",
       "107  [intro]\\n{*mike will made it*}\\nyo.. t.r.u.! (...   \n",
       "108  [intro: r&b samples]\\nnothing in the whole wid...   \n",
       "109  [intro]\\nyuck daddy! yuck!\\nyuck daddy! yuck!\\...   \n",
       "110  [intro]\\ngoo-oh-oh-ohhh {*2x*}\\n(c'mon people!...   \n",
       "111  [intro/chorus - interpolation of krs-one]\\nthe...   \n",
       "112  [8thw1]\\nthe teacher, class in session\\nthe be...   \n",
       "113  [8thw1]\\nwhy stress mic skills? i upgrade to l...   \n",
       "114  [mr. mixx]\\ncheck it out y'all\\nch-ch-check it...   \n",
       "115  ( *mr. mixx cuts in dolemite* )\\n(i wouldn't l...   \n",
       "116  [ fresh kid ice ]\\nlisten up y'all 'cause this...   \n",
       "117  chorus(4x):  luke\\nsomebody say hey-y-y-y-y we...   \n",
       "118  1-2 1-2\\ni am the dominating fresh kid ice\\nan...   \n",
       "119  [note:  brother marquis does all the verses.]\\...   \n",
       "\n",
       "                                song  \n",
       "105                          Like Me  \n",
       "106                    Money Machine  \n",
       "107                           No Lie  \n",
       "108                      Stop Me Now  \n",
       "109                            Yuck!  \n",
       "110                          More Go  \n",
       "111                  Short and Sweet  \n",
       "112                Say My Name Right  \n",
       "113                           Talkin  \n",
       "114               Check it Out Y'all  \n",
       "115                      Get it Girl  \n",
       "116                    Throw the 'D'  \n",
       "117               We Want Some Pussy  \n",
       "118  2 Live is What We Are... (Word)  \n",
       "119                  My Seven Bizzos  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rap_data = pd.DataFrame(df_data)\n",
    "rap_data.iloc[105:120]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(33371, 4)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rap_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking Words\n",
    "===================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50 Cent  -  Leave the Lights On\n",
      "Ab-Soul f/ A-Mack, Punch, SZA  -  Dub Sac\n",
      "Action Bronson f/ Big Body Bes, Mac Miller  -  Twin Peugots\n",
      "The Ambassador f/ LaKia Wise  -  Get You Open\n",
      "Azealia Banks f/ MJ Cole, Peter Rosenberg  -  Desperado\n",
      "Beast 1333  -  113 Bars\n",
      "Beast 1333  -  Anonymous\n",
      "Big Daddy Kane  -  Uncut, Pure (Original and Remix)\n",
      "Childish Gambino (Donald Glover)  -  Lights Turned On\n",
      "D-12  -  American Psycho\n",
      "Deltron 3030 (Del the Funky Homosapien)  -  Battle Song\n",
      "Fat Joe f/ Busta Rhymes, DJ Khaled, Jadakiss, Miguel, Mos Def, Roscoe Dash, Kanye West  -  Pride N Joy\n",
      "G. Dep f/ Kool G. Rap, Rakim  -  I Am\n",
      "Ghostface Killah, Masta Killa, U-God, Raekwon, Cappadonna  -  Winter Warz\n",
      "Hell Razah f/ Fokis, Killah Priest  -  Gladiators\n",
      "Ice-T f/ EJ Evil E the Great, Trigger Tha Gambler (SMG)  -  Cramp Your Style (Live)\n",
      "Ice-T  -  Cramp Your Style\n",
      "Joe Budden  -  Roll Call\n",
      "Killah Priest f/ Antonio Chance  -  Atoms to Adams\n",
      "DJ KaySlay f/ Fat Joe, Raekwon, Scarface  -  I Never Liked Ya Ass\n",
      "DJ KaySlay f/ LL Cool J  -  The Truth\n",
      "Kreators f/ Akrobatik, Big Shug, Ed O.G., Guru, Krumb Snatcha  -  Home\n",
      "Kurupt f/ Deadly Venoms  -  It's Time\t\n",
      "Lil Twist f/ Busta Rhymes  -  Turnt Up\n",
      "Lil Wayne  -  Sh!t\n",
      "Lil Wayne f/ B.G., Manny Fresh  -  Drop it Like it's Hot\n",
      "LL Cool J  -  Murdergram \n",
      "Ludacris  -  Dancin Dirty\n",
      "MellowHype f/ Mike G  -  666\n",
      "R. Kelly f/ Big Tigger, Cam'Ron  -  Snake (Remix)\n",
      "Missy Elliott f/ Timbaland  -  9th Inning\n",
      "Nicki Minaj f/ Skylar Grey  -  Bed of Lies\n",
      "OutKast f/ Lil Wayne, Snoop Dogg  -  Hollywood Divorce\n",
      "Raekwon f/ American Cream Team, Killa Sin  -  Giant Size\n",
      "Raekwon f/ Lord Superb  -  Russian Cut\n",
      "Lord Have Mercy and D.V. alias Khrist  -  Holy Water\n",
      "The Union f/ Buckshot and Kasino  -  Throw it Up\n",
      "Rick Ross  -  Crocodile Python\n",
      "The Roots  -  Quills\n",
      "Erick Sermon f/ Ryze, Twon Gabz  -  Fix Your Face\n",
      "Swollen Members f/ Chali 2na, Evidence  -  Full Contact\n",
      "The Game  -  400 Bars\n",
      "The LOX f/ Foxy Brown  -  My Niggaz\n",
      "Twista f/ Do or Die, Johnny P  -  Yo Body\n",
      "Wyclef Jean f/ Supreme C, Marie Antoinette, Hope  -  Da Cypha\n",
      "Xzibit f/ Nate Dogg  -  Been a Long Time\n"
     ]
    }
   ],
   "source": [
    "for artist, song, lyrics in zip(df_data[\"artist\"],\n",
    "                                df_data[\"song\"],\n",
    "                                df_data[\"lyrics\"]):\n",
    "    if \"python\" in set(lyrics.split()):\n",
    "        print(artist, \" - \", song)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2Pac f/ Outlawz  -  Runnin On E\n",
      "2nd II None f/ DJ Quik, AMG, Hi-C, Playa Hamm  -  Got A Nu Woman\n",
      "All City  -  Ded Right\n",
      "Big Ed  -  Head Busta\n",
      "Big K.R.I.T. f/ Ludacris  -  What U Mean\n",
      "Big L f/ Stan Spit  -  Who You Slidin' Wit'\n",
      "Big Punisher  -  The Dream Shatterer\n",
      "Big Punisher  -  The Dream Shatterer (Original)\n",
      "Canibus  -  Who Stopped Ya?\n",
      "Chamillionaire  -  Set it Off Freestyle\n",
      "Chamillionaire f/ Rasaq  -  Panky Rang\n",
      "Clika One f/ Kurupt, Chiko Dateh, Don Cisco  -  Gangsta Pimpin'\n",
      "Danny!  -  D.A.N.N.Y.\n",
      "Daz Dillinger & JT the Bigga Figga f/ Kurupt, Rappin' 4-Tay  -  Sweet Love\n",
      "E-40  -  If If Was a 5th\n",
      "Flo Rida f/ Robin Thicke, Verdine White  -  I Don't Like It, I Love It\n",
      "Jim Crow f/ Jazze Pha, Too $hort  -  That Drama (Baby's Momma)\n",
      "Juvenile f/ Jay Da Menace, Kango Slim  -  Make U Feel Alright\n",
      "Kanye West f/ 2 Chainz, Marsha Ambrosius, Big Sean, James Fauntleroy II  -  The One\n",
      "Kool G Rap  -  A Thug's Love Story (Chapter I, II, III)\n",
      "Kurupt f/ Lil 1/2 Dead  -  On, OnSite\n",
      "LL Cool J f/ Mashonda, Swizz Beatz  -  Crazy Girl\n",
      "Hollister, Dave f/ Redman, Erick Sermon  -  The Weekend\n",
      "Nicki Minaj  -  Anaconda\n",
      "OutKast  -  War\n",
      "P. Diddy (Puff Daddy) f/ Eightball & MJG, Faith Evans  -  Roll With Me\n",
      "Raekwon  -  Respect Power\n",
      "Royce Da 5'9\" f/ Mr. Porter  -  Mine In Thiz\n",
      "Silkk the Shocker f/ No Limit All-Stars  -  I'm a Soldier\n",
      "Slaughterhouse  -  SayDatThen\n",
      "Soulja Boy  -  Go to War\n",
      "Tech N9ne f/ Shabba Ranks  -  Boy Toy\n",
      "Tekitha f/ Cappadonna  -  I Can See\n",
      "The Game f/ Future, Young Jeezy  -  I Remember *\n",
      "Tyga f/ Future, The Game  -  I Remember\n",
      "Xzibit  -  A Minute to Pray\n"
     ]
    }
   ],
   "source": [
    "for artist, song, lyrics in zip(df_data[\"artist\"],\n",
    "                                df_data[\"song\"],\n",
    "                                df_data[\"lyrics\"]):\n",
    "    if \"anaconda\" in set(lyrics.split()):\n",
    "        print(artist, \" - \", song)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finding Rare Words\n",
    "=================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>333313</th>\n",
       "      <td>goooglo</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333314</th>\n",
       "      <td>gooogla</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333315</th>\n",
       "      <td>gooogd</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333316</th>\n",
       "      <td>gooofa</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333317</th>\n",
       "      <td>goooao</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333318</th>\n",
       "      <td>goollo</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333319</th>\n",
       "      <td>goolld</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333320</th>\n",
       "      <td>goolh</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333321</th>\n",
       "      <td>goolgee</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333322</th>\n",
       "      <td>googook</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333323</th>\n",
       "      <td>googllr</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333324</th>\n",
       "      <td>googlal</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333325</th>\n",
       "      <td>googgoo</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333326</th>\n",
       "      <td>googgol</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333327</th>\n",
       "      <td>goofel</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333328</th>\n",
       "      <td>gooek</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333329</th>\n",
       "      <td>gooddg</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333330</th>\n",
       "      <td>gooblle</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333331</th>\n",
       "      <td>gollgo</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333332</th>\n",
       "      <td>golgw</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           word  count\n",
       "333313  goooglo  12711\n",
       "333314  gooogla  12711\n",
       "333315   gooogd  12711\n",
       "333316   gooofa  12711\n",
       "333317   goooao  12711\n",
       "333318   goollo  12711\n",
       "333319   goolld  12711\n",
       "333320    goolh  12711\n",
       "333321  goolgee  12711\n",
       "333322  googook  12711\n",
       "333323  googllr  12711\n",
       "333324  googlal  12711\n",
       "333325  googgoo  12711\n",
       "333326  googgol  12711\n",
       "333327   goofel  12711\n",
       "333328    gooek  12711\n",
       "333329   gooddg  12711\n",
       "333330  gooblle  12711\n",
       "333331   gollgo  12711\n",
       "333332    golgw  12711"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The 1/3 million most frequent words, all lowercase, with counts. \n",
    "# http://norvig.com/ngrams/\n",
    "ngrams=pd.read_csv(\"assets/count_1w.txt\",sep=\"\\t\",names=[\"word\",\"count\"])\n",
    "ngrams.tail(n=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# The Tournament Word List (178,690 words) -- used by North American Scrabble players.\n",
    "# http://norvig.com/ngrams/\n",
    "twl_wordlist=pd.read_csv(\"assets/TWL06.txt\",names=[\"word\"])\n",
    "twl_wordlist=pd.DataFrame(twl_wordlist[\"word\"].str.lower())\n",
    "\n",
    "twl_w_ngrams = pd.merge(twl_wordlist, ngrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>73785</th>\n",
       "      <td>triose</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59922</th>\n",
       "      <td>retinae</td>\n",
       "      <td>12712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35948</th>\n",
       "      <td>inhibitive</td>\n",
       "      <td>12714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57100</th>\n",
       "      <td>rasbora</td>\n",
       "      <td>12714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40668</th>\n",
       "      <td>loadstar</td>\n",
       "      <td>12714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2765</th>\n",
       "      <td>antemortem</td>\n",
       "      <td>12714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45565</th>\n",
       "      <td>munting</td>\n",
       "      <td>12714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72551</th>\n",
       "      <td>toits</td>\n",
       "      <td>12716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11210</th>\n",
       "      <td>cestus</td>\n",
       "      <td>12716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36139</th>\n",
       "      <td>insatiably</td>\n",
       "      <td>12717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18170</th>\n",
       "      <td>defoliate</td>\n",
       "      <td>12718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41780</th>\n",
       "      <td>malaprops</td>\n",
       "      <td>12719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70256</th>\n",
       "      <td>synchronises</td>\n",
       "      <td>12719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64308</th>\n",
       "      <td>sibilance</td>\n",
       "      <td>12720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70772</th>\n",
       "      <td>tappets</td>\n",
       "      <td>12723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40563</th>\n",
       "      <td>lither</td>\n",
       "      <td>12724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27791</th>\n",
       "      <td>forfend</td>\n",
       "      <td>12724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73589</th>\n",
       "      <td>treponemal</td>\n",
       "      <td>12724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48890</th>\n",
       "      <td>outcross</td>\n",
       "      <td>12725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16895</th>\n",
       "      <td>cuprum</td>\n",
       "      <td>12728</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               word  count\n",
       "73785        triose  12711\n",
       "59922       retinae  12712\n",
       "35948    inhibitive  12714\n",
       "57100       rasbora  12714\n",
       "40668      loadstar  12714\n",
       "2765     antemortem  12714\n",
       "45565       munting  12714\n",
       "72551         toits  12716\n",
       "11210        cestus  12716\n",
       "36139    insatiably  12717\n",
       "18170     defoliate  12718\n",
       "41780     malaprops  12719\n",
       "70256  synchronises  12719\n",
       "64308     sibilance  12720\n",
       "70772       tappets  12723\n",
       "40563        lither  12724\n",
       "27791       forfend  12724\n",
       "73589    treponemal  12724\n",
       "48890      outcross  12725\n",
       "16895        cuprum  12728"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twl_w_ngrams.sort_values(by=\"count\").head(n=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "51195"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(twl_w_ngrams[\"count\"] < 400000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['aardvarks', 'aardwolf', 'aargh', 'abaca', 'aback', 'abaft', 'abandonments', 'abandons', 'abase', 'abased', 'abasement', 'abashed', 'abated', 'abatements', 'abates', 'abating', 'abattoir', 'abattoirs', 'abaxial', 'abaya', 'abbes', 'abbess', 'abbesses', 'abbeys', 'abbots', 'abbreviate', 'abbreviates', 'abbreviating', 'abdicate', 'abdicated', 'abdicates', 'abdicating', 'abdication', 'abdomens', 'abdominals', 'abduct', 'abductee', 'abductees', 'abducting', 'abductions', 'abductor', 'abductors', 'abducts', 'abeam', 'abecedarian', 'abele', 'abeles', 'abelia', 'aberrant', 'abets', 'abetted', 'abetting', 'abettor', 'abeyance', 'abhor', 'abhorred', 'abhorrence', 'abhorrent', 'abhors', 'abided', 'abides', 'abiogenesis', 'abiotic', 'abject', 'abjection', 'abjectly', 'abjuration', 'abjure', 'abjured', 'ablate', 'ablated', 'ablations', 'ablative', 'ablaze', 'abled', 'abler', 'ables', 'ablest', 'ablution', 'ablutions', 'abnegation', 'abodes', 'abolishes', 'abolishing', 'abolishment', 'abolitionism', 'abolitionist', 'abolitionists', 'abominable', 'abominably', 'abomination', 'abominations', 'aboriginals', 'aborigine', 'aborigines', 'abortifacient', 'aborting', 'abortionist', 'abortionists', 'abortive']\n"
     ]
    }
   ],
   "source": [
    "all_bigwords = twl_w_ngrams[twl_w_ngrams[\"count\"] < 400000][\"word\"].values\n",
    "all_big_bigwords = [word for word in all_bigwords if len(word) > 4]\n",
    "bigset = set(all_big_bigwords)\n",
    "\n",
    "print(all_big_bigwords[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_data[\"matches\"] = []\n",
    "for lyrics in df_data[\"lyrics\"]:\n",
    "    found_words = set(lyrics.split()) & bigset\n",
    "    df_data[\"matches\"].append(found_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rap_data = pd.DataFrame(df_data)\n",
    "rap_data = rap_data[rap_data.matches != set()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist</th>\n",
       "      <th>filename</th>\n",
       "      <th>lyrics</th>\n",
       "      <th>matches</th>\n",
       "      <th>song</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10 K.A.N.'s</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/10_kans/rm_bside/u_...</td>\n",
       "      <td>[* applause *]\\n\\n[dj]\\nright about now we got...</td>\n",
       "      <td>{fellas, skeet, fiends}</td>\n",
       "      <td>U Need Dick In Your Life</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10sion</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/10sion/tension/lets...</td>\n",
       "      <td>-=talking=-\\nlets get it on every time\\nholler...</td>\n",
       "      <td>{playas, holler, hypnotized}</td>\n",
       "      <td>Let's Get it On</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>113</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/113/dans_lur/ausumm...</td>\n",
       "      <td>* send corrections to the typist\\n\\n[refrain]\\...</td>\n",
       "      <td>{favelas, typist, prise, pavillon, spliff, dom...</td>\n",
       "      <td>Au Summum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1200 Techniques</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1200tech/c_theory/e...</td>\n",
       "      <td>* send corrections to the typist\\n\\n*scratchin...</td>\n",
       "      <td>{typist, showdowns, shipwrecked, emcees, sicke...</td>\n",
       "      <td>Eye of the Storm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1200 Techniques</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1200tech/c_theory/w...</td>\n",
       "      <td>[dj peril]\\nnfamous - step, step, step, step, ...</td>\n",
       "      <td>{melanin, renegades, booed, befriending, albin...</td>\n",
       "      <td>Where Ur At?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1200 Techniques</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1200tech/infinite/k...</td>\n",
       "      <td>* send corrections to the typist\\n\\n[prolouge]...</td>\n",
       "      <td>{typist, showdowns, hopscotch, quitters, fickl...</td>\n",
       "      <td>Karma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>12 O'Clock w/ Raekwon the Chef</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/12oclock/rm_bside/n...</td>\n",
       "      <td>intro: raekwon the chef \\n\\nyeah yeah, that's ...</td>\n",
       "      <td>{jubilant, boneyard, livest, beefs, gleam, swo...</td>\n",
       "      <td>Nasty Immigrants *</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1982 (Statik Selektah &amp; Termanology) f/ Cassid...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1982/1982/goinback....</td>\n",
       "      <td>(i'm goin back, back, back)\\n\\n[verse 1 - cass...</td>\n",
       "      <td>{backer, iliad, porky, caskets}</td>\n",
       "      <td>Goin Back</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1982 (Statik Selektah &amp; Termanology) f/ Lil' F...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1982/rm_bside/thuga...</td>\n",
       "      <td>[intro]\\nbrrrrrrrrrrrrrrrrr\\nshow off, show of...</td>\n",
       "      <td>{hugger, mobsters}</td>\n",
       "      <td>Thugathon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.4.0. Productions f/ Chapel</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/po_poets/f...</td>\n",
       "      <td>[intro: chapel]\\nlet me bless this shit, i'mma...</td>\n",
       "      <td>{fiends, deadliest, cocking}</td>\n",
       "      <td>Freestyle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.4.0. Productions f/ Franky Botts, Molly-Q</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/po_poets/g...</td>\n",
       "      <td>[molly-q]\\ndart double click, through your car...</td>\n",
       "      <td>{detonate, godfathers, plutons}</td>\n",
       "      <td>Godfathers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.4.0. Productions f/ Cheesey Rat, Crunch Lo, ...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/po_poets/g...</td>\n",
       "      <td>\"only the gods could watch the earth twist\" -&gt;...</td>\n",
       "      <td>{willies, caramels, nobly, wheezy, realest}</td>\n",
       "      <td>God Twist</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1.4.0. Productions f/ Franky Botts, Othorized ...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/po_poets/s...</td>\n",
       "      <td>[intro: lounge lo]\\nya'll know the work, aiyo\\...</td>\n",
       "      <td>{lounging, escapade, stutter, stubbed, perfect...</td>\n",
       "      <td>Salute</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.4.0. Productions f/ Crunch Lo</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/po_poets/s...</td>\n",
       "      <td>[hook: crunch lo]\\nis it the thug? deep in my ...</td>\n",
       "      <td>{embezzle, bugged, reefer, corns}</td>\n",
       "      <td>So What Now?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.4.0. Productions f/ Othorized F.A.M.</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/po_poets/w...</td>\n",
       "      <td>[shawn wigs]\\ndon't believe the hype, we hold ...</td>\n",
       "      <td>{ratchets}</td>\n",
       "      <td>We Always Gonna Do It</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1.4.0. Productions f/ Crunch Lo, Samantha</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/e...</td>\n",
       "      <td>[chorus: samantha (crunch lo)]\\ni will give yo...</td>\n",
       "      <td>{timberlands, prowl, beaters, extort, jakes, s...</td>\n",
       "      <td>Everything U Need</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1.4.0. Productions f/ Molly-Q</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/f...</td>\n",
       "      <td>[molly-q]\\nballistic, linguistic, slang-tistic...</td>\n",
       "      <td>{twats, tippy}</td>\n",
       "      <td>Freestyle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1.4.0. Productions f/ Shawn Wigs</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/k...</td>\n",
       "      <td>[intro: shawn wigs]\\nson, it was the illest fu...</td>\n",
       "      <td>{stashed, illest, spasm}</td>\n",
       "      <td>Knee Deep in Some Ass</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1.4.0. Productions f/ Franky Botts, Lighter Sh...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/l...</td>\n",
       "      <td>[chorus 2x: lighter shade]\\nlet it bang, bang ...</td>\n",
       "      <td>{pillage, wiseguys, televise, manholes, shogun...</td>\n",
       "      <td>Let it Bang</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1.4.0. Productions f/ Shawn Wigs, Toilet</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/l...</td>\n",
       "      <td>[chorus: toilet (shawn wigs)]\\ni got a mansion...</td>\n",
       "      <td>{smalls, shoeshine, maxed}</td>\n",
       "      <td>Life's Been Good to Me</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               artist  \\\n",
       "0                                         10 K.A.N.'s   \n",
       "1                                              10sion   \n",
       "2                                                 113   \n",
       "3                                     1200 Techniques   \n",
       "4                                     1200 Techniques   \n",
       "5                                     1200 Techniques   \n",
       "6                     12 O'Clock w/ Raekwon the Chef    \n",
       "8   1982 (Statik Selektah & Termanology) f/ Cassid...   \n",
       "9   1982 (Statik Selektah & Termanology) f/ Lil' F...   \n",
       "10                       1.4.0. Productions f/ Chapel   \n",
       "11        1.4.0. Productions f/ Franky Botts, Molly-Q   \n",
       "12  1.4.0. Productions f/ Cheesey Rat, Crunch Lo, ...   \n",
       "13  1.4.0. Productions f/ Franky Botts, Othorized ...   \n",
       "14                    1.4.0. Productions f/ Crunch Lo   \n",
       "15             1.4.0. Productions f/ Othorized F.A.M.   \n",
       "16          1.4.0. Productions f/ Crunch Lo, Samantha   \n",
       "17                      1.4.0. Productions f/ Molly-Q   \n",
       "18                   1.4.0. Productions f/ Shawn Wigs   \n",
       "19  1.4.0. Productions f/ Franky Botts, Lighter Sh...   \n",
       "20           1.4.0. Productions f/ Shawn Wigs, Toilet   \n",
       "\n",
       "                                             filename  \\\n",
       "0   Lyrics/ohhla.com/anonymous/10_kans/rm_bside/u_...   \n",
       "1   Lyrics/ohhla.com/anonymous/10sion/tension/lets...   \n",
       "2   Lyrics/ohhla.com/anonymous/113/dans_lur/ausumm...   \n",
       "3   Lyrics/ohhla.com/anonymous/1200tech/c_theory/e...   \n",
       "4   Lyrics/ohhla.com/anonymous/1200tech/c_theory/w...   \n",
       "5   Lyrics/ohhla.com/anonymous/1200tech/infinite/k...   \n",
       "6   Lyrics/ohhla.com/anonymous/12oclock/rm_bside/n...   \n",
       "8   Lyrics/ohhla.com/anonymous/1982/1982/goinback....   \n",
       "9   Lyrics/ohhla.com/anonymous/1982/rm_bside/thuga...   \n",
       "10  Lyrics/ohhla.com/anonymous/1pt_four/po_poets/f...   \n",
       "11  Lyrics/ohhla.com/anonymous/1pt_four/po_poets/g...   \n",
       "12  Lyrics/ohhla.com/anonymous/1pt_four/po_poets/g...   \n",
       "13  Lyrics/ohhla.com/anonymous/1pt_four/po_poets/s...   \n",
       "14  Lyrics/ohhla.com/anonymous/1pt_four/po_poets/s...   \n",
       "15  Lyrics/ohhla.com/anonymous/1pt_four/po_poets/w...   \n",
       "16  Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/e...   \n",
       "17  Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/f...   \n",
       "18  Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/k...   \n",
       "19  Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/l...   \n",
       "20  Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/l...   \n",
       "\n",
       "                                               lyrics  \\\n",
       "0   [* applause *]\\n\\n[dj]\\nright about now we got...   \n",
       "1   -=talking=-\\nlets get it on every time\\nholler...   \n",
       "2   * send corrections to the typist\\n\\n[refrain]\\...   \n",
       "3   * send corrections to the typist\\n\\n*scratchin...   \n",
       "4   [dj peril]\\nnfamous - step, step, step, step, ...   \n",
       "5   * send corrections to the typist\\n\\n[prolouge]...   \n",
       "6   intro: raekwon the chef \\n\\nyeah yeah, that's ...   \n",
       "8   (i'm goin back, back, back)\\n\\n[verse 1 - cass...   \n",
       "9   [intro]\\nbrrrrrrrrrrrrrrrrr\\nshow off, show of...   \n",
       "10  [intro: chapel]\\nlet me bless this shit, i'mma...   \n",
       "11  [molly-q]\\ndart double click, through your car...   \n",
       "12  \"only the gods could watch the earth twist\" ->...   \n",
       "13  [intro: lounge lo]\\nya'll know the work, aiyo\\...   \n",
       "14  [hook: crunch lo]\\nis it the thug? deep in my ...   \n",
       "15  [shawn wigs]\\ndon't believe the hype, we hold ...   \n",
       "16  [chorus: samantha (crunch lo)]\\ni will give yo...   \n",
       "17  [molly-q]\\nballistic, linguistic, slang-tistic...   \n",
       "18  [intro: shawn wigs]\\nson, it was the illest fu...   \n",
       "19  [chorus 2x: lighter shade]\\nlet it bang, bang ...   \n",
       "20  [chorus: toilet (shawn wigs)]\\ni got a mansion...   \n",
       "\n",
       "                                              matches  \\\n",
       "0                             {fellas, skeet, fiends}   \n",
       "1                        {playas, holler, hypnotized}   \n",
       "2   {favelas, typist, prise, pavillon, spliff, dom...   \n",
       "3   {typist, showdowns, shipwrecked, emcees, sicke...   \n",
       "4   {melanin, renegades, booed, befriending, albin...   \n",
       "5   {typist, showdowns, hopscotch, quitters, fickl...   \n",
       "6   {jubilant, boneyard, livest, beefs, gleam, swo...   \n",
       "8                     {backer, iliad, porky, caskets}   \n",
       "9                                  {hugger, mobsters}   \n",
       "10                       {fiends, deadliest, cocking}   \n",
       "11                    {detonate, godfathers, plutons}   \n",
       "12        {willies, caramels, nobly, wheezy, realest}   \n",
       "13  {lounging, escapade, stutter, stubbed, perfect...   \n",
       "14                  {embezzle, bugged, reefer, corns}   \n",
       "15                                         {ratchets}   \n",
       "16  {timberlands, prowl, beaters, extort, jakes, s...   \n",
       "17                                     {twats, tippy}   \n",
       "18                           {stashed, illest, spasm}   \n",
       "19  {pillage, wiseguys, televise, manholes, shogun...   \n",
       "20                         {smalls, shoeshine, maxed}   \n",
       "\n",
       "                        song  \n",
       "0   U Need Dick In Your Life  \n",
       "1            Let's Get it On  \n",
       "2                  Au Summum  \n",
       "3           Eye of the Storm  \n",
       "4               Where Ur At?  \n",
       "5                      Karma  \n",
       "6        Nasty Immigrants *   \n",
       "8                  Goin Back  \n",
       "9                  Thugathon  \n",
       "10                 Freestyle  \n",
       "11                Godfathers  \n",
       "12                 God Twist  \n",
       "13                    Salute  \n",
       "14              So What Now?  \n",
       "15     We Always Gonna Do It  \n",
       "16         Everything U Need  \n",
       "17                 Freestyle  \n",
       "18     Knee Deep in Some Ass  \n",
       "19               Let it Bang  \n",
       "20    Life's Been Good to Me  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rap_data.head(n=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finding Rare Rap Words\n",
    "======================"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "rap_onegrams = Counter()\n",
    "\n",
    "for lyrics in df_data[\"lyrics\"]:\n",
    "    rap_onegrams.update(lyrics.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rap_onegrams_df = pd.DataFrame.from_dict(rap_onegrams, orient='index').reset_index()\n",
    "rap_onegrams_df.columns = [\"word\",\"count\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>192727</th>\n",
       "      <td>is</td>\n",
       "      <td>112697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36746</th>\n",
       "      <td>get</td>\n",
       "      <td>112745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39709</th>\n",
       "      <td>we</td>\n",
       "      <td>129374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360668</th>\n",
       "      <td>your</td>\n",
       "      <td>131125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155504</th>\n",
       "      <td>with</td>\n",
       "      <td>137393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110645</th>\n",
       "      <td>of</td>\n",
       "      <td>141163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6432</th>\n",
       "      <td>like</td>\n",
       "      <td>151442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146271</th>\n",
       "      <td>that</td>\n",
       "      <td>163367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138161</th>\n",
       "      <td>on</td>\n",
       "      <td>174972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200839</th>\n",
       "      <td>me</td>\n",
       "      <td>175888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274697</th>\n",
       "      <td>i'm</td>\n",
       "      <td>189356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122193</th>\n",
       "      <td>it</td>\n",
       "      <td>208014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118228</th>\n",
       "      <td>in</td>\n",
       "      <td>226361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173041</th>\n",
       "      <td>my</td>\n",
       "      <td>254126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87779</th>\n",
       "      <td>to</td>\n",
       "      <td>341578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223350</th>\n",
       "      <td>and</td>\n",
       "      <td>366264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20481</th>\n",
       "      <td>a</td>\n",
       "      <td>374886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171794</th>\n",
       "      <td>you</td>\n",
       "      <td>423922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9205</th>\n",
       "      <td>i</td>\n",
       "      <td>492419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22813</th>\n",
       "      <td>the</td>\n",
       "      <td>742326</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        word   count\n",
       "192727    is  112697\n",
       "36746    get  112745\n",
       "39709     we  129374\n",
       "360668  your  131125\n",
       "155504  with  137393\n",
       "110645    of  141163\n",
       "6432    like  151442\n",
       "146271  that  163367\n",
       "138161    on  174972\n",
       "200839    me  175888\n",
       "274697   i'm  189356\n",
       "122193    it  208014\n",
       "118228    in  226361\n",
       "173041    my  254126\n",
       "87779     to  341578\n",
       "223350   and  366264\n",
       "20481      a  374886\n",
       "171794   you  423922\n",
       "9205       i  492419\n",
       "22813    the  742326"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rap_onegrams_df.sort_values(by=\"count\").tail(n=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19515"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raptwl_df = pd.merge(twl_wordlist, rap_onegrams_df, on=\"word\")\n",
    "raptwl_df.head()\n",
    "\n",
    "all_bigrapwords = set(raptwl_df[raptwl_df[\"count\"] < 5][\"word\"].values)\n",
    "len(all_bigrapwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_data[\"rap_matches\"] = []\n",
    "for lyrics in df_data[\"lyrics\"]:\n",
    "    found_words = set(lyrics.split()) & all_bigrapwords\n",
    "    df_data[\"rap_matches\"].append(found_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist</th>\n",
       "      <th>filename</th>\n",
       "      <th>lyrics</th>\n",
       "      <th>matches</th>\n",
       "      <th>rap_matches</th>\n",
       "      <th>song</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>113</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/113/dans_lur/ausumm...</td>\n",
       "      <td>* send corrections to the typist\\n\\n[refrain]\\...</td>\n",
       "      <td>{favelas, typist, prise, pavillon, spliff, dom...</td>\n",
       "      <td>{cartes, favelas, prise, pavillon, haute, domi...</td>\n",
       "      <td>Au Summum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1200 Techniques</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1200tech/c_theory/e...</td>\n",
       "      <td>* send corrections to the typist\\n\\n*scratchin...</td>\n",
       "      <td>{typist, showdowns, shipwrecked, emcees, sicke...</td>\n",
       "      <td>{shipwrecked}</td>\n",
       "      <td>Eye of the Storm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1200 Techniques</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1200tech/c_theory/w...</td>\n",
       "      <td>[dj peril]\\nnfamous - step, step, step, step, ...</td>\n",
       "      <td>{melanin, renegades, booed, befriending, albin...</td>\n",
       "      <td>{albinos, befriending, constructing, swooned}</td>\n",
       "      <td>Where Ur At?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1200 Techniques</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1200tech/infinite/k...</td>\n",
       "      <td>* send corrections to the typist\\n\\n[prolouge]...</td>\n",
       "      <td>{typist, showdowns, hopscotch, quitters, fickl...</td>\n",
       "      <td>{dodgy, umbilicals}</td>\n",
       "      <td>Karma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>12 O'Clock w/ Raekwon the Chef</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/12oclock/rm_bside/n...</td>\n",
       "      <td>intro: raekwon the chef \\n\\nyeah yeah, that's ...</td>\n",
       "      <td>{jubilant, boneyard, livest, beefs, gleam, swo...</td>\n",
       "      <td>{jubilant, sweetened, boneyard, connives}</td>\n",
       "      <td>Nasty Immigrants *</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1982 (Statik Selektah &amp; Termanology) f/ Cassid...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1982/1982/goinback....</td>\n",
       "      <td>(i'm goin back, back, back)\\n\\n[verse 1 - cass...</td>\n",
       "      <td>{backer, iliad, porky, caskets}</td>\n",
       "      <td>{iliad}</td>\n",
       "      <td>Goin Back</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1982 (Statik Selektah &amp; Termanology) f/ Lil' F...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1982/rm_bside/thuga...</td>\n",
       "      <td>[intro]\\nbrrrrrrrrrrrrrrrrr\\nshow off, show of...</td>\n",
       "      <td>{hugger, mobsters}</td>\n",
       "      <td>{hugger}</td>\n",
       "      <td>Thugathon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.4.0. Productions f/ Chapel</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/po_poets/f...</td>\n",
       "      <td>[intro: chapel]\\nlet me bless this shit, i'mma...</td>\n",
       "      <td>{fiends, deadliest, cocking}</td>\n",
       "      <td>{cloves}</td>\n",
       "      <td>Freestyle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.4.0. Productions f/ Franky Botts, Molly-Q</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/po_poets/g...</td>\n",
       "      <td>[molly-q]\\ndart double click, through your car...</td>\n",
       "      <td>{detonate, godfathers, plutons}</td>\n",
       "      <td>{plutons, sirloins}</td>\n",
       "      <td>Godfathers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.4.0. Productions f/ Cheesey Rat, Crunch Lo, ...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/po_poets/g...</td>\n",
       "      <td>\"only the gods could watch the earth twist\" -&gt;...</td>\n",
       "      <td>{willies, caramels, nobly, wheezy, realest}</td>\n",
       "      <td>{caramels, nobly}</td>\n",
       "      <td>God Twist</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1.4.0. Productions f/ Franky Botts, Othorized ...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/po_poets/s...</td>\n",
       "      <td>[intro: lounge lo]\\nya'll know the work, aiyo\\...</td>\n",
       "      <td>{lounging, escapade, stutter, stubbed, perfect...</td>\n",
       "      <td>{goombah, raviolis, bulked}</td>\n",
       "      <td>Salute</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.4.0. Productions f/ Crunch Lo</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/po_poets/s...</td>\n",
       "      <td>[hook: crunch lo]\\nis it the thug? deep in my ...</td>\n",
       "      <td>{embezzle, bugged, reefer, corns}</td>\n",
       "      <td>{ameba, dismissing, toughly}</td>\n",
       "      <td>So What Now?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.4.0. Productions f/ Othorized F.A.M.</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/po_poets/w...</td>\n",
       "      <td>[shawn wigs]\\ndon't believe the hype, we hold ...</td>\n",
       "      <td>{ratchets}</td>\n",
       "      <td>{breasted}</td>\n",
       "      <td>We Always Gonna Do It</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1.4.0. Productions f/ Franky Botts, Lighter Sh...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/l...</td>\n",
       "      <td>[chorus 2x: lighter shade]\\nlet it bang, bang ...</td>\n",
       "      <td>{pillage, wiseguys, televise, manholes, shogun...</td>\n",
       "      <td>{croquets, manholes, gondola}</td>\n",
       "      <td>Let it Bang</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1.4.0. Productions f/ Chapel, Wise</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/l...</td>\n",
       "      <td>[chorus 2x: wise]\\nyo, we unfold, leaving you ...</td>\n",
       "      <td>{plucking, proclamations, sware, snitch}</td>\n",
       "      <td>{proclamations}</td>\n",
       "      <td>Lock N Load</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1.4.0. Productions f/ Clocka, Lighter Shade, O...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/n...</td>\n",
       "      <td>[intro: molly-q]\\nput your hands against the w...</td>\n",
       "      <td>{bandanas, menudo, stashed, heavyweights}</td>\n",
       "      <td>{barnyards}</td>\n",
       "      <td>New York</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1.4.0. Productions f/ Crunch Lo, Shawn Wigs</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/o...</td>\n",
       "      <td>[chorus: shawn wigs (crunch lo)]\\nwho am i? (w...</td>\n",
       "      <td>{corns, bargained, pillage, antonym, offspring...</td>\n",
       "      <td>{offsprings}</td>\n",
       "      <td>One on One</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1.4.0. Productions f/ Cheesey Rat, Crunch Lo, ...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/r...</td>\n",
       "      <td>[cheesey rat]\\nwhat time is it, i checked the ...</td>\n",
       "      <td>{blockhead, bruise, sneeze, bugged, clogging, ...</td>\n",
       "      <td>{administrations, reinventing, wised, particip...</td>\n",
       "      <td>Revelations</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1.4.0. Productions f/ Othorized F.A.M.</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/s...</td>\n",
       "      <td>[lounge lo]\\ni'm in the jungle walking barefoo...</td>\n",
       "      <td>{lounger, scuffed, automatics, splurge, lucked...</td>\n",
       "      <td>{lounger, lulu, epilogues}</td>\n",
       "      <td>Staten Islanders</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1.4.0. Productions f/ Crunch Lo, Lighter Shade</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/w...</td>\n",
       "      <td>[lighter shade]\\nya'll want to battle, so i'mm...</td>\n",
       "      <td>{scottie, jezebel, pillage, brainchild, chins,...</td>\n",
       "      <td>{blitzing, spindle, equivalence}</td>\n",
       "      <td>We in the Realm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1.4.0. Productions f/ Chapel, Livewire, Wise</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/stand_up/d...</td>\n",
       "      <td>[chorus: wise]\\nthe block is hot, hot, son, be...</td>\n",
       "      <td>{copped, gassed, polluters, fittest}</td>\n",
       "      <td>{polluters}</td>\n",
       "      <td>Da Block is Hot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1.4.0. Productions f/ Shawn Wigs</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/stand_up/d...</td>\n",
       "      <td>\"drop!\"\\n\\n[shawn wigs]\\nyou should be thankfu...</td>\n",
       "      <td>{coked, bounties, slumped, snotty}</td>\n",
       "      <td>{regs, bounties, socialist}</td>\n",
       "      <td>Drop!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>1.4.0. Productions f/ Molly-Q, Shawn Wigs</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/stand_up/p...</td>\n",
       "      <td>[shawn wigs]\\nwe've been on the map, since gho...</td>\n",
       "      <td>{expels, snorting, hollows, prowl}</td>\n",
       "      <td>{expels}</td>\n",
       "      <td>Poisonous Poets</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>1.4.0. Productions f/ Othorized F.A.M.</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/stand_up/s...</td>\n",
       "      <td>[intro: lounge lo]\\nthat's right.. i see you\\n...</td>\n",
       "      <td>{dipper, muscled, clapped, quitter, reefer, th...</td>\n",
       "      <td>{breachers, muscled, thoroughness}</td>\n",
       "      <td>Shot Me Down</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>1.4.0. Productions f/ King Just</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/stand_up/t...</td>\n",
       "      <td>[hook: king just singing]\\nhey yo, you got mad...</td>\n",
       "      <td>{interferer, poltergeists, vindicate, windpipe}</td>\n",
       "      <td>{interferer, vindicate}</td>\n",
       "      <td>This is Hip-Hop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1.4.0. Productions f/ Crunch Lo</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1pt_four/stand_up/t...</td>\n",
       "      <td>[intro: crunch lo]\\nit ain't nothing, son, wor...</td>\n",
       "      <td>{corns, foreheads, goons, yapping, waltzed}</td>\n",
       "      <td>{waltzed}</td>\n",
       "      <td>Trouble to MC's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>1st Infantry f/ Prodigy</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/1st_infa/rm_bside/s...</td>\n",
       "      <td>[prodigy]\\ngangsta, yeah nigga you know (you k...</td>\n",
       "      <td>{roaches, reefer, sovereignly}</td>\n",
       "      <td>{sovereignly}</td>\n",
       "      <td>Serious (The New Message)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>213</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/213/hard_way/joysti...</td>\n",
       "      <td>[nate dogg]\\nshe said she wants to ride my joy...</td>\n",
       "      <td>{}</td>\n",
       "      <td>{joysticks}</td>\n",
       "      <td>Joystick</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>213</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/213/hard_way/maryja...</td>\n",
       "      <td>[chorus - nate dogg]\\none for the treble, two ...</td>\n",
       "      <td>{sipped, hooka, swisher}</td>\n",
       "      <td>{respectful}</td>\n",
       "      <td>Mary Jane</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>2 Chainz (Tity Boi)</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/2_chainz/2residue/b...</td>\n",
       "      <td>[chorus: x2]\\ni just call her boo\\ni don't kno...</td>\n",
       "      <td>{layin, daycares, reefer, chirp}</td>\n",
       "      <td>{mien, daycares}</td>\n",
       "      <td>Boo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33294</th>\n",
       "      <td>Young Scooter f/ Bun B</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yscooter/lottery/st...</td>\n",
       "      <td>[chorus]\\ngrowing up as a young nigga\\nyoung n...</td>\n",
       "      <td>{noter, snitch, franklins}</td>\n",
       "      <td>{noter}</td>\n",
       "      <td>Street Lottery</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33299</th>\n",
       "      <td>Young Scooter f/ Alley Boy, Gucci Mane</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yscooter/voice_of/f...</td>\n",
       "      <td>[chorus]\\nfaster\\nhe did five, he got chill mo...</td>\n",
       "      <td>{rapping}</td>\n",
       "      <td>{bulled}</td>\n",
       "      <td>Faster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33301</th>\n",
       "      <td>Yukmouth f/ Tech N9ne</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yukmouth/thugged/bu...</td>\n",
       "      <td>[tech n9ne] (yukmouth)\\nregime killas, ah\\nwha...</td>\n",
       "      <td>{geeked, strangulated, loony, woofers, fiends,...</td>\n",
       "      <td>{clunk, strangulated}</td>\n",
       "      <td>Bumbell</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33302</th>\n",
       "      <td>Yukmouth f/ Rap-A-Lot Family</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yukmouth/thugged/RA...</td>\n",
       "      <td>[yukmouth]\\ni live the life of a hoodlum \\ntak...</td>\n",
       "      <td>{wiseguys, mobbed, riffles, caskets, layin, hu...</td>\n",
       "      <td>{blackbirds, welly, spore, riffles, whooshes}</td>\n",
       "      <td>RAL Mafia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33303</th>\n",
       "      <td>Yukmouth f/ N.W.A, Tech N9ne</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yukmouth/thugged/st...</td>\n",
       "      <td>[dr. dre] \\nthe motherfucking saga continues.....</td>\n",
       "      <td>{guzzled, bauble, motherfucking, straddle, mot...</td>\n",
       "      <td>{bauble, manipulative, caver}</td>\n",
       "      <td>Stallion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33304</th>\n",
       "      <td>Yukmouth f/ Madd Maxx, Phats Bossalini, Poppa ...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yukmouth/thugged/th...</td>\n",
       "      <td>yo! let's do this!\\nuh!\\nuh-oh, uh-oh!\\nha!\\nu...</td>\n",
       "      <td>{tatted, caulked, blindfold, timberlands, spit...</td>\n",
       "      <td>{callas, clocker, caulked}</td>\n",
       "      <td>Thugged Out</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33306</th>\n",
       "      <td>Yukmouth f/ Phats Bossilini, Mad Max, Tech N9n...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yukmouth/thuglord/r...</td>\n",
       "      <td>[ chorus ]\\ni got some killers on the payroll,...</td>\n",
       "      <td>{blowed, tatted, crackhead, sowed, splatter, p...</td>\n",
       "      <td>{patrolled}</td>\n",
       "      <td>Regime Killers 2001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33307</th>\n",
       "      <td>Yukmouth f/ Kokane, Kurupt, Nate Dogg</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yukmouth/thuglord/s...</td>\n",
       "      <td>[nate dogg]\\neverytime i smoke\\ni smoke indo s...</td>\n",
       "      <td>{snitch, motherfuckers, subtractions, homies, ...</td>\n",
       "      <td>{subtractions}</td>\n",
       "      <td>So Ignorant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33308</th>\n",
       "      <td>Yukmouth f/ Gonzoe, Kris Kaliko, Tech N9ne</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yukmouth/unitedg2/k...</td>\n",
       "      <td>[intro: gonzoe]\\nyes yes, whuttup mayn\\nit's t...</td>\n",
       "      <td>{sawed, scrappers, gunshots, ambushed, cocked,...</td>\n",
       "      <td>{scrappers}</td>\n",
       "      <td>Kill 'Em Off</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33310</th>\n",
       "      <td>Yung Joc f/ Rick Ross, Snoop Dogg</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yung_joc/hustlen/br...</td>\n",
       "      <td>[snoop dogg] + (yung joc)\\nyung joc (sup?)\\nwh...</td>\n",
       "      <td>{sideshows, bezels, sneaks}</td>\n",
       "      <td>{sideshows}</td>\n",
       "      <td>Brand New</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33312</th>\n",
       "      <td>Yung Joc f/ The Game, Jim Jones</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yung_joc/hustlen/c_...</td>\n",
       "      <td>[the game] (yung joc)\\nhey joc (what up nigga?...</td>\n",
       "      <td>{pumper, chirp, asthmatics, fiends, moguls, in...</td>\n",
       "      <td>{asthmatics}</td>\n",
       "      <td>Cut Throat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33313</th>\n",
       "      <td>Yung Joc f/ Jazze Pha, Trick Daddy</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yung_joc/hustlen/ch...</td>\n",
       "      <td>{*screw voice: \"hustlenomic$!\"*}\\n\\n[intro: tr...</td>\n",
       "      <td>{fishbowl, antifreeze, bougie, eights, minks, ...</td>\n",
       "      <td>{fishbowl}</td>\n",
       "      <td>Chevy Smile</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33315</th>\n",
       "      <td>Yung Joc f/ Mike Carlito, Gorilla Zoe</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yung_joc/hustlen/ge...</td>\n",
       "      <td>[intro/chorus - samples]\\ni be getting to da m...</td>\n",
       "      <td>{medulla, snitches, takin, earlobe}</td>\n",
       "      <td>{derringers}</td>\n",
       "      <td>Getting to Da Money</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33319</th>\n",
       "      <td>Yung Joc f/ Southerngirl</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yung_joc/hustlen/lv...</td>\n",
       "      <td>[chorus: southerngirl] + (yung joc)\\ni'm just,...</td>\n",
       "      <td>{broads, baller, takin}</td>\n",
       "      <td>{bistro}</td>\n",
       "      <td>Living the Life</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33328</th>\n",
       "      <td>Yung Joc</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yung_joc/newjcity/g...</td>\n",
       "      <td>* first single; send corrections to the typist...</td>\n",
       "      <td>{typist, duces, playmaker, nitty, coups, mothe...</td>\n",
       "      <td>{playmaker}</td>\n",
       "      <td>It's Goin' Down</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33329</th>\n",
       "      <td>Yung Joc</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yung_joc/newjcity/h...</td>\n",
       "      <td>[chorus: repeat 2x]\\nall the hoes lose they mi...</td>\n",
       "      <td>{fiends, flinch, swang, headland, homies, reefer}</td>\n",
       "      <td>{headland}</td>\n",
       "      <td>Hear Me Coming</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33340</th>\n",
       "      <td>Yung La f/ T.I., Young Dro</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yung_la/rm_bside/ai...</td>\n",
       "      <td>[verse 1 - yung la]\\nyou got to know your boy ...</td>\n",
       "      <td>{commandeer, nauseous}</td>\n",
       "      <td>{commandeer}</td>\n",
       "      <td>Ain't I (Remix)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33341</th>\n",
       "      <td>Yung Ro f/ Chamillionaire, D-Black, Chamillion...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yung_ro/nob_land/ya...</td>\n",
       "      <td>* send corrections to the typist\\n\\n[mixing by...</td>\n",
       "      <td>{typist, scuff, sicker, hotshot, blacked, inte...</td>\n",
       "      <td>{hotshot}</td>\n",
       "      <td>Y'all Don't Want Us to Come!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33346</th>\n",
       "      <td>Yung Berg f/ Jim Jones, Junior, Rich Boy</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/yungberg/a_famous/s...</td>\n",
       "      <td>[yung berg]\\nyeah - so you know we had to do a...</td>\n",
       "      <td>{adios, woolies, yellowed}</td>\n",
       "      <td>{woolies, yellowed}</td>\n",
       "      <td>Sexy Lady (Remix)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33349</th>\n",
       "      <td>Z-Ro f/ K-Rino</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/z_ro/angeldst/today...</td>\n",
       "      <td>(*talking*)\\nuh oh, z-ro the crooked\\nand my m...</td>\n",
       "      <td>{chastises, forgave, hearse}</td>\n",
       "      <td>{chastises, amoxicillin}</td>\n",
       "      <td>Today</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33351</th>\n",
       "      <td>Z-Ro f/ Bun B, P.O.P.</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/z_ro/im_still/remem...</td>\n",
       "      <td>(*talking*)\\nhell yeah nigga, ay yo\\n\\n[z-ro]\\...</td>\n",
       "      <td>{sported, egghead, ridiculed, rapping}</td>\n",
       "      <td>{egghead}</td>\n",
       "      <td>Remember Me</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33354</th>\n",
       "      <td>Z-Ro f/ Willie D</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/z_ro/meth/onemotme....</td>\n",
       "      <td>(*talking*)\\nso you getting married tomorrow h...</td>\n",
       "      <td>{bowlegs, butterball, homeboy, dicking}</td>\n",
       "      <td>{bowlegs}</td>\n",
       "      <td>One Mo Time</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33357</th>\n",
       "      <td>Z-Ro f/ Scarface</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/z_ro/the_life/t_nig...</td>\n",
       "      <td>[hook]\\nthese niggaz, must of forgot\\nthat i w...</td>\n",
       "      <td>{denting, motherfucking, vaseline, snitch}</td>\n",
       "      <td>{denting}</td>\n",
       "      <td>These Niggaz</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33362</th>\n",
       "      <td>Z-Ro f/ Daz, Thug Dirt</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/z_ro/toleranc/time_...</td>\n",
       "      <td>(*talking*)\\nyeah, let me count that up right ...</td>\n",
       "      <td>{barging, lounging, swerving, slanging, saggin...</td>\n",
       "      <td>{trudging, uncock}</td>\n",
       "      <td>Time and Time Again</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33363</th>\n",
       "      <td>Zion I &amp; The Grouch feat. Chali 2na</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/zion_i/heroesin/too...</td>\n",
       "      <td>[verse 1-chali 2na]\\nyeah, some of us are caug...</td>\n",
       "      <td>{devilish, noose, thumpers, scantily, recluse,...</td>\n",
       "      <td>{quakers, scantily}</td>\n",
       "      <td>Too Much</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33364</th>\n",
       "      <td>Zion I f/ Talib Kweli</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/zion_i/true_and/tem...</td>\n",
       "      <td>[zion i]\\nyeah, yeah, hey\\nthe cityscape where...</td>\n",
       "      <td>{slurring, pimped, emulating, confessing, popu...</td>\n",
       "      <td>{cityscape, populating}</td>\n",
       "      <td>Temperature</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33366</th>\n",
       "      <td>Zo! f/ Eric Roberson, Phonte</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/zo/justvis3/marzipa...</td>\n",
       "      <td>[verse one: eric roberson]\\nto know you is to ...</td>\n",
       "      <td>{marzipan}</td>\n",
       "      <td>{marzipan}</td>\n",
       "      <td>Marzipan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33368</th>\n",
       "      <td>Zo! f/ Rapper Big Pooh, Darien Brockington, Er...</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/zo/sunstorm/thscoul...</td>\n",
       "      <td>{say brother, do you see all these foxys out h...</td>\n",
       "      <td>{innocently, cocked}</td>\n",
       "      <td>{innocently}</td>\n",
       "      <td>This Could Be the Night</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33369</th>\n",
       "      <td>Zoe Pound f/ Raekwon</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/zoepound/bullethd/b...</td>\n",
       "      <td>[intro: raekwon]\\ndon't play, what, what, you ...</td>\n",
       "      <td>{menses}</td>\n",
       "      <td>{insiders, menses}</td>\n",
       "      <td>Back to the Cages</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33370</th>\n",
       "      <td>Zu Ninjaz f/ Drunken Dragon, Ol' Dirty Bastard</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/zuninjaz/now_just/s...</td>\n",
       "      <td>[sample - repeated in background]\\nstanding.. ...</td>\n",
       "      <td>{chumps, mathematic, shits, lyrically}</td>\n",
       "      <td>{bloodstreams}</td>\n",
       "      <td>Specially Trained Ninja</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16461 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  artist  \\\n",
       "2                                                    113   \n",
       "3                                        1200 Techniques   \n",
       "4                                        1200 Techniques   \n",
       "5                                        1200 Techniques   \n",
       "6                        12 O'Clock w/ Raekwon the Chef    \n",
       "8      1982 (Statik Selektah & Termanology) f/ Cassid...   \n",
       "9      1982 (Statik Selektah & Termanology) f/ Lil' F...   \n",
       "10                          1.4.0. Productions f/ Chapel   \n",
       "11           1.4.0. Productions f/ Franky Botts, Molly-Q   \n",
       "12     1.4.0. Productions f/ Cheesey Rat, Crunch Lo, ...   \n",
       "13     1.4.0. Productions f/ Franky Botts, Othorized ...   \n",
       "14                       1.4.0. Productions f/ Crunch Lo   \n",
       "15                1.4.0. Productions f/ Othorized F.A.M.   \n",
       "19     1.4.0. Productions f/ Franky Botts, Lighter Sh...   \n",
       "21                    1.4.0. Productions f/ Chapel, Wise   \n",
       "22     1.4.0. Productions f/ Clocka, Lighter Shade, O...   \n",
       "23           1.4.0. Productions f/ Crunch Lo, Shawn Wigs   \n",
       "24     1.4.0. Productions f/ Cheesey Rat, Crunch Lo, ...   \n",
       "25                1.4.0. Productions f/ Othorized F.A.M.   \n",
       "26        1.4.0. Productions f/ Crunch Lo, Lighter Shade   \n",
       "29          1.4.0. Productions f/ Chapel, Livewire, Wise   \n",
       "30                      1.4.0. Productions f/ Shawn Wigs   \n",
       "33             1.4.0. Productions f/ Molly-Q, Shawn Wigs   \n",
       "34                1.4.0. Productions f/ Othorized F.A.M.   \n",
       "35                       1.4.0. Productions f/ King Just   \n",
       "36                       1.4.0. Productions f/ Crunch Lo   \n",
       "40                               1st Infantry f/ Prodigy   \n",
       "46                                                   213   \n",
       "50                                                   213   \n",
       "57                                   2 Chainz (Tity Boi)   \n",
       "...                                                  ...   \n",
       "33294                             Young Scooter f/ Bun B   \n",
       "33299             Young Scooter f/ Alley Boy, Gucci Mane   \n",
       "33301                              Yukmouth f/ Tech N9ne   \n",
       "33302                       Yukmouth f/ Rap-A-Lot Family   \n",
       "33303                       Yukmouth f/ N.W.A, Tech N9ne   \n",
       "33304  Yukmouth f/ Madd Maxx, Phats Bossalini, Poppa ...   \n",
       "33306  Yukmouth f/ Phats Bossilini, Mad Max, Tech N9n...   \n",
       "33307              Yukmouth f/ Kokane, Kurupt, Nate Dogg   \n",
       "33308         Yukmouth f/ Gonzoe, Kris Kaliko, Tech N9ne   \n",
       "33310                  Yung Joc f/ Rick Ross, Snoop Dogg   \n",
       "33312                    Yung Joc f/ The Game, Jim Jones   \n",
       "33313                 Yung Joc f/ Jazze Pha, Trick Daddy   \n",
       "33315              Yung Joc f/ Mike Carlito, Gorilla Zoe   \n",
       "33319                           Yung Joc f/ Southerngirl   \n",
       "33328                                           Yung Joc   \n",
       "33329                                           Yung Joc   \n",
       "33340                         Yung La f/ T.I., Young Dro   \n",
       "33341  Yung Ro f/ Chamillionaire, D-Black, Chamillion...   \n",
       "33346           Yung Berg f/ Jim Jones, Junior, Rich Boy   \n",
       "33349                                     Z-Ro f/ K-Rino   \n",
       "33351                              Z-Ro f/ Bun B, P.O.P.   \n",
       "33354                                   Z-Ro f/ Willie D   \n",
       "33357                                   Z-Ro f/ Scarface   \n",
       "33362                             Z-Ro f/ Daz, Thug Dirt   \n",
       "33363                Zion I & The Grouch feat. Chali 2na   \n",
       "33364                              Zion I f/ Talib Kweli   \n",
       "33366                       Zo! f/ Eric Roberson, Phonte   \n",
       "33368  Zo! f/ Rapper Big Pooh, Darien Brockington, Er...   \n",
       "33369                               Zoe Pound f/ Raekwon   \n",
       "33370     Zu Ninjaz f/ Drunken Dragon, Ol' Dirty Bastard   \n",
       "\n",
       "                                                filename  \\\n",
       "2      Lyrics/ohhla.com/anonymous/113/dans_lur/ausumm...   \n",
       "3      Lyrics/ohhla.com/anonymous/1200tech/c_theory/e...   \n",
       "4      Lyrics/ohhla.com/anonymous/1200tech/c_theory/w...   \n",
       "5      Lyrics/ohhla.com/anonymous/1200tech/infinite/k...   \n",
       "6      Lyrics/ohhla.com/anonymous/12oclock/rm_bside/n...   \n",
       "8      Lyrics/ohhla.com/anonymous/1982/1982/goinback....   \n",
       "9      Lyrics/ohhla.com/anonymous/1982/rm_bside/thuga...   \n",
       "10     Lyrics/ohhla.com/anonymous/1pt_four/po_poets/f...   \n",
       "11     Lyrics/ohhla.com/anonymous/1pt_four/po_poets/g...   \n",
       "12     Lyrics/ohhla.com/anonymous/1pt_four/po_poets/g...   \n",
       "13     Lyrics/ohhla.com/anonymous/1pt_four/po_poets/s...   \n",
       "14     Lyrics/ohhla.com/anonymous/1pt_four/po_poets/s...   \n",
       "15     Lyrics/ohhla.com/anonymous/1pt_four/po_poets/w...   \n",
       "19     Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/l...   \n",
       "21     Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/l...   \n",
       "22     Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/n...   \n",
       "23     Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/o...   \n",
       "24     Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/r...   \n",
       "25     Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/s...   \n",
       "26     Lyrics/ohhla.com/anonymous/1pt_four/skoolyrd/w...   \n",
       "29     Lyrics/ohhla.com/anonymous/1pt_four/stand_up/d...   \n",
       "30     Lyrics/ohhla.com/anonymous/1pt_four/stand_up/d...   \n",
       "33     Lyrics/ohhla.com/anonymous/1pt_four/stand_up/p...   \n",
       "34     Lyrics/ohhla.com/anonymous/1pt_four/stand_up/s...   \n",
       "35     Lyrics/ohhla.com/anonymous/1pt_four/stand_up/t...   \n",
       "36     Lyrics/ohhla.com/anonymous/1pt_four/stand_up/t...   \n",
       "40     Lyrics/ohhla.com/anonymous/1st_infa/rm_bside/s...   \n",
       "46     Lyrics/ohhla.com/anonymous/213/hard_way/joysti...   \n",
       "50     Lyrics/ohhla.com/anonymous/213/hard_way/maryja...   \n",
       "57     Lyrics/ohhla.com/anonymous/2_chainz/2residue/b...   \n",
       "...                                                  ...   \n",
       "33294  Lyrics/ohhla.com/anonymous/yscooter/lottery/st...   \n",
       "33299  Lyrics/ohhla.com/anonymous/yscooter/voice_of/f...   \n",
       "33301  Lyrics/ohhla.com/anonymous/yukmouth/thugged/bu...   \n",
       "33302  Lyrics/ohhla.com/anonymous/yukmouth/thugged/RA...   \n",
       "33303  Lyrics/ohhla.com/anonymous/yukmouth/thugged/st...   \n",
       "33304  Lyrics/ohhla.com/anonymous/yukmouth/thugged/th...   \n",
       "33306  Lyrics/ohhla.com/anonymous/yukmouth/thuglord/r...   \n",
       "33307  Lyrics/ohhla.com/anonymous/yukmouth/thuglord/s...   \n",
       "33308  Lyrics/ohhla.com/anonymous/yukmouth/unitedg2/k...   \n",
       "33310  Lyrics/ohhla.com/anonymous/yung_joc/hustlen/br...   \n",
       "33312  Lyrics/ohhla.com/anonymous/yung_joc/hustlen/c_...   \n",
       "33313  Lyrics/ohhla.com/anonymous/yung_joc/hustlen/ch...   \n",
       "33315  Lyrics/ohhla.com/anonymous/yung_joc/hustlen/ge...   \n",
       "33319  Lyrics/ohhla.com/anonymous/yung_joc/hustlen/lv...   \n",
       "33328  Lyrics/ohhla.com/anonymous/yung_joc/newjcity/g...   \n",
       "33329  Lyrics/ohhla.com/anonymous/yung_joc/newjcity/h...   \n",
       "33340  Lyrics/ohhla.com/anonymous/yung_la/rm_bside/ai...   \n",
       "33341  Lyrics/ohhla.com/anonymous/yung_ro/nob_land/ya...   \n",
       "33346  Lyrics/ohhla.com/anonymous/yungberg/a_famous/s...   \n",
       "33349  Lyrics/ohhla.com/anonymous/z_ro/angeldst/today...   \n",
       "33351  Lyrics/ohhla.com/anonymous/z_ro/im_still/remem...   \n",
       "33354  Lyrics/ohhla.com/anonymous/z_ro/meth/onemotme....   \n",
       "33357  Lyrics/ohhla.com/anonymous/z_ro/the_life/t_nig...   \n",
       "33362  Lyrics/ohhla.com/anonymous/z_ro/toleranc/time_...   \n",
       "33363  Lyrics/ohhla.com/anonymous/zion_i/heroesin/too...   \n",
       "33364  Lyrics/ohhla.com/anonymous/zion_i/true_and/tem...   \n",
       "33366  Lyrics/ohhla.com/anonymous/zo/justvis3/marzipa...   \n",
       "33368  Lyrics/ohhla.com/anonymous/zo/sunstorm/thscoul...   \n",
       "33369  Lyrics/ohhla.com/anonymous/zoepound/bullethd/b...   \n",
       "33370  Lyrics/ohhla.com/anonymous/zuninjaz/now_just/s...   \n",
       "\n",
       "                                                  lyrics  \\\n",
       "2      * send corrections to the typist\\n\\n[refrain]\\...   \n",
       "3      * send corrections to the typist\\n\\n*scratchin...   \n",
       "4      [dj peril]\\nnfamous - step, step, step, step, ...   \n",
       "5      * send corrections to the typist\\n\\n[prolouge]...   \n",
       "6      intro: raekwon the chef \\n\\nyeah yeah, that's ...   \n",
       "8      (i'm goin back, back, back)\\n\\n[verse 1 - cass...   \n",
       "9      [intro]\\nbrrrrrrrrrrrrrrrrr\\nshow off, show of...   \n",
       "10     [intro: chapel]\\nlet me bless this shit, i'mma...   \n",
       "11     [molly-q]\\ndart double click, through your car...   \n",
       "12     \"only the gods could watch the earth twist\" ->...   \n",
       "13     [intro: lounge lo]\\nya'll know the work, aiyo\\...   \n",
       "14     [hook: crunch lo]\\nis it the thug? deep in my ...   \n",
       "15     [shawn wigs]\\ndon't believe the hype, we hold ...   \n",
       "19     [chorus 2x: lighter shade]\\nlet it bang, bang ...   \n",
       "21     [chorus 2x: wise]\\nyo, we unfold, leaving you ...   \n",
       "22     [intro: molly-q]\\nput your hands against the w...   \n",
       "23     [chorus: shawn wigs (crunch lo)]\\nwho am i? (w...   \n",
       "24     [cheesey rat]\\nwhat time is it, i checked the ...   \n",
       "25     [lounge lo]\\ni'm in the jungle walking barefoo...   \n",
       "26     [lighter shade]\\nya'll want to battle, so i'mm...   \n",
       "29     [chorus: wise]\\nthe block is hot, hot, son, be...   \n",
       "30     \"drop!\"\\n\\n[shawn wigs]\\nyou should be thankfu...   \n",
       "33     [shawn wigs]\\nwe've been on the map, since gho...   \n",
       "34     [intro: lounge lo]\\nthat's right.. i see you\\n...   \n",
       "35     [hook: king just singing]\\nhey yo, you got mad...   \n",
       "36     [intro: crunch lo]\\nit ain't nothing, son, wor...   \n",
       "40     [prodigy]\\ngangsta, yeah nigga you know (you k...   \n",
       "46     [nate dogg]\\nshe said she wants to ride my joy...   \n",
       "50     [chorus - nate dogg]\\none for the treble, two ...   \n",
       "57     [chorus: x2]\\ni just call her boo\\ni don't kno...   \n",
       "...                                                  ...   \n",
       "33294  [chorus]\\ngrowing up as a young nigga\\nyoung n...   \n",
       "33299  [chorus]\\nfaster\\nhe did five, he got chill mo...   \n",
       "33301  [tech n9ne] (yukmouth)\\nregime killas, ah\\nwha...   \n",
       "33302  [yukmouth]\\ni live the life of a hoodlum \\ntak...   \n",
       "33303  [dr. dre] \\nthe motherfucking saga continues.....   \n",
       "33304  yo! let's do this!\\nuh!\\nuh-oh, uh-oh!\\nha!\\nu...   \n",
       "33306  [ chorus ]\\ni got some killers on the payroll,...   \n",
       "33307  [nate dogg]\\neverytime i smoke\\ni smoke indo s...   \n",
       "33308  [intro: gonzoe]\\nyes yes, whuttup mayn\\nit's t...   \n",
       "33310  [snoop dogg] + (yung joc)\\nyung joc (sup?)\\nwh...   \n",
       "33312  [the game] (yung joc)\\nhey joc (what up nigga?...   \n",
       "33313  {*screw voice: \"hustlenomic$!\"*}\\n\\n[intro: tr...   \n",
       "33315  [intro/chorus - samples]\\ni be getting to da m...   \n",
       "33319  [chorus: southerngirl] + (yung joc)\\ni'm just,...   \n",
       "33328  * first single; send corrections to the typist...   \n",
       "33329  [chorus: repeat 2x]\\nall the hoes lose they mi...   \n",
       "33340  [verse 1 - yung la]\\nyou got to know your boy ...   \n",
       "33341  * send corrections to the typist\\n\\n[mixing by...   \n",
       "33346  [yung berg]\\nyeah - so you know we had to do a...   \n",
       "33349  (*talking*)\\nuh oh, z-ro the crooked\\nand my m...   \n",
       "33351  (*talking*)\\nhell yeah nigga, ay yo\\n\\n[z-ro]\\...   \n",
       "33354  (*talking*)\\nso you getting married tomorrow h...   \n",
       "33357  [hook]\\nthese niggaz, must of forgot\\nthat i w...   \n",
       "33362  (*talking*)\\nyeah, let me count that up right ...   \n",
       "33363  [verse 1-chali 2na]\\nyeah, some of us are caug...   \n",
       "33364  [zion i]\\nyeah, yeah, hey\\nthe cityscape where...   \n",
       "33366  [verse one: eric roberson]\\nto know you is to ...   \n",
       "33368  {say brother, do you see all these foxys out h...   \n",
       "33369  [intro: raekwon]\\ndon't play, what, what, you ...   \n",
       "33370  [sample - repeated in background]\\nstanding.. ...   \n",
       "\n",
       "                                                 matches  \\\n",
       "2      {favelas, typist, prise, pavillon, spliff, dom...   \n",
       "3      {typist, showdowns, shipwrecked, emcees, sicke...   \n",
       "4      {melanin, renegades, booed, befriending, albin...   \n",
       "5      {typist, showdowns, hopscotch, quitters, fickl...   \n",
       "6      {jubilant, boneyard, livest, beefs, gleam, swo...   \n",
       "8                        {backer, iliad, porky, caskets}   \n",
       "9                                     {hugger, mobsters}   \n",
       "10                          {fiends, deadliest, cocking}   \n",
       "11                       {detonate, godfathers, plutons}   \n",
       "12           {willies, caramels, nobly, wheezy, realest}   \n",
       "13     {lounging, escapade, stutter, stubbed, perfect...   \n",
       "14                     {embezzle, bugged, reefer, corns}   \n",
       "15                                            {ratchets}   \n",
       "19     {pillage, wiseguys, televise, manholes, shogun...   \n",
       "21              {plucking, proclamations, sware, snitch}   \n",
       "22             {bandanas, menudo, stashed, heavyweights}   \n",
       "23     {corns, bargained, pillage, antonym, offspring...   \n",
       "24     {blockhead, bruise, sneeze, bugged, clogging, ...   \n",
       "25     {lounger, scuffed, automatics, splurge, lucked...   \n",
       "26     {scottie, jezebel, pillage, brainchild, chins,...   \n",
       "29                  {copped, gassed, polluters, fittest}   \n",
       "30                    {coked, bounties, slumped, snotty}   \n",
       "33                    {expels, snorting, hollows, prowl}   \n",
       "34     {dipper, muscled, clapped, quitter, reefer, th...   \n",
       "35       {interferer, poltergeists, vindicate, windpipe}   \n",
       "36           {corns, foreheads, goons, yapping, waltzed}   \n",
       "40                        {roaches, reefer, sovereignly}   \n",
       "46                                                    {}   \n",
       "50                              {sipped, hooka, swisher}   \n",
       "57                      {layin, daycares, reefer, chirp}   \n",
       "...                                                  ...   \n",
       "33294                         {noter, snitch, franklins}   \n",
       "33299                                          {rapping}   \n",
       "33301  {geeked, strangulated, loony, woofers, fiends,...   \n",
       "33302  {wiseguys, mobbed, riffles, caskets, layin, hu...   \n",
       "33303  {guzzled, bauble, motherfucking, straddle, mot...   \n",
       "33304  {tatted, caulked, blindfold, timberlands, spit...   \n",
       "33306  {blowed, tatted, crackhead, sowed, splatter, p...   \n",
       "33307  {snitch, motherfuckers, subtractions, homies, ...   \n",
       "33308  {sawed, scrappers, gunshots, ambushed, cocked,...   \n",
       "33310                        {sideshows, bezels, sneaks}   \n",
       "33312  {pumper, chirp, asthmatics, fiends, moguls, in...   \n",
       "33313  {fishbowl, antifreeze, bougie, eights, minks, ...   \n",
       "33315                {medulla, snitches, takin, earlobe}   \n",
       "33319                            {broads, baller, takin}   \n",
       "33328  {typist, duces, playmaker, nitty, coups, mothe...   \n",
       "33329  {fiends, flinch, swang, headland, homies, reefer}   \n",
       "33340                             {commandeer, nauseous}   \n",
       "33341  {typist, scuff, sicker, hotshot, blacked, inte...   \n",
       "33346                         {adios, woolies, yellowed}   \n",
       "33349                       {chastises, forgave, hearse}   \n",
       "33351             {sported, egghead, ridiculed, rapping}   \n",
       "33354            {bowlegs, butterball, homeboy, dicking}   \n",
       "33357         {denting, motherfucking, vaseline, snitch}   \n",
       "33362  {barging, lounging, swerving, slanging, saggin...   \n",
       "33363  {devilish, noose, thumpers, scantily, recluse,...   \n",
       "33364  {slurring, pimped, emulating, confessing, popu...   \n",
       "33366                                         {marzipan}   \n",
       "33368                               {innocently, cocked}   \n",
       "33369                                           {menses}   \n",
       "33370             {chumps, mathematic, shits, lyrically}   \n",
       "\n",
       "                                             rap_matches  \\\n",
       "2      {cartes, favelas, prise, pavillon, haute, domi...   \n",
       "3                                          {shipwrecked}   \n",
       "4          {albinos, befriending, constructing, swooned}   \n",
       "5                                    {dodgy, umbilicals}   \n",
       "6              {jubilant, sweetened, boneyard, connives}   \n",
       "8                                                {iliad}   \n",
       "9                                               {hugger}   \n",
       "10                                              {cloves}   \n",
       "11                                   {plutons, sirloins}   \n",
       "12                                     {caramels, nobly}   \n",
       "13                           {goombah, raviolis, bulked}   \n",
       "14                          {ameba, dismissing, toughly}   \n",
       "15                                            {breasted}   \n",
       "19                         {croquets, manholes, gondola}   \n",
       "21                                       {proclamations}   \n",
       "22                                           {barnyards}   \n",
       "23                                          {offsprings}   \n",
       "24     {administrations, reinventing, wised, particip...   \n",
       "25                            {lounger, lulu, epilogues}   \n",
       "26                      {blitzing, spindle, equivalence}   \n",
       "29                                           {polluters}   \n",
       "30                           {regs, bounties, socialist}   \n",
       "33                                              {expels}   \n",
       "34                    {breachers, muscled, thoroughness}   \n",
       "35                               {interferer, vindicate}   \n",
       "36                                             {waltzed}   \n",
       "40                                         {sovereignly}   \n",
       "46                                           {joysticks}   \n",
       "50                                          {respectful}   \n",
       "57                                      {mien, daycares}   \n",
       "...                                                  ...   \n",
       "33294                                            {noter}   \n",
       "33299                                           {bulled}   \n",
       "33301                              {clunk, strangulated}   \n",
       "33302      {blackbirds, welly, spore, riffles, whooshes}   \n",
       "33303                      {bauble, manipulative, caver}   \n",
       "33304                         {callas, clocker, caulked}   \n",
       "33306                                        {patrolled}   \n",
       "33307                                     {subtractions}   \n",
       "33308                                        {scrappers}   \n",
       "33310                                        {sideshows}   \n",
       "33312                                       {asthmatics}   \n",
       "33313                                         {fishbowl}   \n",
       "33315                                       {derringers}   \n",
       "33319                                           {bistro}   \n",
       "33328                                        {playmaker}   \n",
       "33329                                         {headland}   \n",
       "33340                                       {commandeer}   \n",
       "33341                                          {hotshot}   \n",
       "33346                                {woolies, yellowed}   \n",
       "33349                           {chastises, amoxicillin}   \n",
       "33351                                          {egghead}   \n",
       "33354                                          {bowlegs}   \n",
       "33357                                          {denting}   \n",
       "33362                                 {trudging, uncock}   \n",
       "33363                                {quakers, scantily}   \n",
       "33364                            {cityscape, populating}   \n",
       "33366                                         {marzipan}   \n",
       "33368                                       {innocently}   \n",
       "33369                                 {insiders, menses}   \n",
       "33370                                     {bloodstreams}   \n",
       "\n",
       "                               song  \n",
       "2                         Au Summum  \n",
       "3                  Eye of the Storm  \n",
       "4                      Where Ur At?  \n",
       "5                             Karma  \n",
       "6               Nasty Immigrants *   \n",
       "8                         Goin Back  \n",
       "9                         Thugathon  \n",
       "10                        Freestyle  \n",
       "11                       Godfathers  \n",
       "12                        God Twist  \n",
       "13                           Salute  \n",
       "14                     So What Now?  \n",
       "15            We Always Gonna Do It  \n",
       "19                      Let it Bang  \n",
       "21                      Lock N Load  \n",
       "22                         New York  \n",
       "23                       One on One  \n",
       "24                      Revelations  \n",
       "25                 Staten Islanders  \n",
       "26                  We in the Realm  \n",
       "29                  Da Block is Hot  \n",
       "30                            Drop!  \n",
       "33                  Poisonous Poets  \n",
       "34                     Shot Me Down  \n",
       "35                  This is Hip-Hop  \n",
       "36                  Trouble to MC's  \n",
       "40        Serious (The New Message)  \n",
       "46                         Joystick  \n",
       "50                        Mary Jane  \n",
       "57                              Boo  \n",
       "...                             ...  \n",
       "33294                Street Lottery  \n",
       "33299                        Faster  \n",
       "33301                       Bumbell  \n",
       "33302                     RAL Mafia  \n",
       "33303                      Stallion  \n",
       "33304                   Thugged Out  \n",
       "33306           Regime Killers 2001  \n",
       "33307                   So Ignorant  \n",
       "33308                  Kill 'Em Off  \n",
       "33310                     Brand New  \n",
       "33312                    Cut Throat  \n",
       "33313                   Chevy Smile  \n",
       "33315           Getting to Da Money  \n",
       "33319               Living the Life  \n",
       "33328               It's Goin' Down  \n",
       "33329                Hear Me Coming  \n",
       "33340               Ain't I (Remix)  \n",
       "33341  Y'all Don't Want Us to Come!  \n",
       "33346             Sexy Lady (Remix)  \n",
       "33349                         Today  \n",
       "33351                   Remember Me  \n",
       "33354                   One Mo Time  \n",
       "33357                  These Niggaz  \n",
       "33362           Time and Time Again  \n",
       "33363                      Too Much  \n",
       "33364                   Temperature  \n",
       "33366                      Marzipan  \n",
       "33368       This Could Be the Night  \n",
       "33369             Back to the Cages  \n",
       "33370       Specially Trained Ninja  \n",
       "\n",
       "[16461 rows x 6 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rap_data = pd.DataFrame(df_data)\n",
    "rap_data = rap_data[rap_data.rap_matches != set()]\n",
    "rap_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist</th>\n",
       "      <th>filename</th>\n",
       "      <th>lyrics</th>\n",
       "      <th>matches</th>\n",
       "      <th>rap_matches</th>\n",
       "      <th>song</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23373</th>\n",
       "      <td>Nas w/ AZ, The Firm</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/nas/illmatic/genesi...</td>\n",
       "      <td>*sound of a subway train going overhead*\\n*in ...</td>\n",
       "      <td>{coupes, takin, infantryman, trifle}</td>\n",
       "      <td>{infantryman}</td>\n",
       "      <td>The Genesis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23375</th>\n",
       "      <td>Nas</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/nas/illmatic/memory...</td>\n",
       "      <td>\\t(check that shit)\\n\\taight fuck that shit, w...</td>\n",
       "      <td>{dingbats, reminisce, gassed, takin, trifle, s...</td>\n",
       "      <td>{dingbats, physiology}</td>\n",
       "      <td>Memory Lane (Sittin' in Da Park)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23376</th>\n",
       "      <td>Nas</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/nas/illmatic/nysomi...</td>\n",
       "      <td>[intro: nas]\\nyeah yeah, aiyyo black it's time...</td>\n",
       "      <td>{beepers, backtrack, snitch, gunfights, takin,...</td>\n",
       "      <td>{peepholes}</td>\n",
       "      <td>N.Y. State of Mind</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23379</th>\n",
       "      <td>Nas</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/nas/illmatic/rep.na...</td>\n",
       "      <td>represent, represent!! (repeat 4x)\\n\\nstraight...</td>\n",
       "      <td>{guzzle, dissed, dweller, hillbillies, blunts}</td>\n",
       "      <td>{accelerator}</td>\n",
       "      <td>Represent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23380</th>\n",
       "      <td>Nas w/ Pete Rock (uncredited chorus vocals)</td>\n",
       "      <td>Lyrics/ohhla.com/anonymous/nas/illmatic/world_...</td>\n",
       "      <td>\"it's yours!\" --&gt; t la rock\\n\\nchorus: nas, pe...</td>\n",
       "      <td>{amped, caved, phlegm, toothed}</td>\n",
       "      <td>{toothed}</td>\n",
       "      <td>The World is Yours</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            artist  \\\n",
       "23373                          Nas w/ AZ, The Firm   \n",
       "23375                                          Nas   \n",
       "23376                                          Nas   \n",
       "23379                                          Nas   \n",
       "23380  Nas w/ Pete Rock (uncredited chorus vocals)   \n",
       "\n",
       "                                                filename  \\\n",
       "23373  Lyrics/ohhla.com/anonymous/nas/illmatic/genesi...   \n",
       "23375  Lyrics/ohhla.com/anonymous/nas/illmatic/memory...   \n",
       "23376  Lyrics/ohhla.com/anonymous/nas/illmatic/nysomi...   \n",
       "23379  Lyrics/ohhla.com/anonymous/nas/illmatic/rep.na...   \n",
       "23380  Lyrics/ohhla.com/anonymous/nas/illmatic/world_...   \n",
       "\n",
       "                                                  lyrics  \\\n",
       "23373  *sound of a subway train going overhead*\\n*in ...   \n",
       "23375  \\t(check that shit)\\n\\taight fuck that shit, w...   \n",
       "23376  [intro: nas]\\nyeah yeah, aiyyo black it's time...   \n",
       "23379  represent, represent!! (repeat 4x)\\n\\nstraight...   \n",
       "23380  \"it's yours!\" --> t la rock\\n\\nchorus: nas, pe...   \n",
       "\n",
       "                                                 matches  \\\n",
       "23373               {coupes, takin, infantryman, trifle}   \n",
       "23375  {dingbats, reminisce, gassed, takin, trifle, s...   \n",
       "23376  {beepers, backtrack, snitch, gunfights, takin,...   \n",
       "23379     {guzzle, dissed, dweller, hillbillies, blunts}   \n",
       "23380                    {amped, caved, phlegm, toothed}   \n",
       "\n",
       "                  rap_matches                              song  \n",
       "23373           {infantryman}                       The Genesis  \n",
       "23375  {dingbats, physiology}  Memory Lane (Sittin' in Da Park)  \n",
       "23376             {peepholes}                N.Y. State of Mind  \n",
       "23379           {accelerator}                         Represent  \n",
       "23380               {toothed}                The World is Yours  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#rap_data[rap_data[\"artist\"].str.startswith(\"Kanye\")]\n",
    "#rap_data[rap_data[\"artist\"].str.startswith(\"Wu-Tang\") & rap_data[\"filename\"].str.contains(\"enter\")]\n",
    "rap_data[rap_data[\"artist\"].str.startswith(\"Nas\") & rap_data[\"filename\"].str.contains(\"illmatic\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dingbats', 'reminisce', 'gassed', 'takin', 'trifle', 'shoelaces', 'fiends', 'ganja', 'overdoses', 'vexed'}\n",
      "{'dingbats', 'physiology'}\n"
     ]
    }
   ],
   "source": [
    "# Search and replace 23375 to 1173 in the lyrics subset on Github!\n",
    "print(rap_data.loc[23375][\"matches\"])\n",
    "print(rap_data.loc[23375][\"rap_matches\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word = \"trifle\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* * *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finding 'Good' Songs\n",
    "===================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nas  -  Memory Lane (Sittin' in da Park) 52\n"
     ]
    }
   ],
   "source": [
    "import spotipy\n",
    "sp = spotipy.Spotify()\n",
    "    \n",
    "def test_track_search(sp, search_str):\n",
    "    results = sp.search(q=search_str, type='track', limit=1)\n",
    "    if len(results['tracks']['items']) > 0:\n",
    "        print(results['tracks']['items'][0]['artists'][0]['name'],\" - \",\n",
    "              results['tracks']['items'][0]['name'],\n",
    "              results['tracks']['items'][0]['popularity'],)\n",
    "    else:\n",
    "        print(\"\")\n",
    "\n",
    "test_track_search(sp, 'Nas Memory Lane')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finding 'Good' Songs with YouTube\n",
    "================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Activate Google Data API\n",
    "DEVELOPER_KEY = \"\"\n",
    "\n",
    "from apiclient.discovery import build\n",
    "from datetime import datetime\n",
    "\n",
    "def youtube_search(q):\n",
    "  youtube = build(\"youtube\", \"v3\", developerKey=DEVELOPER_KEY)\n",
    "\n",
    "  search_response = youtube.search().list(\n",
    "    q=q, type=\"video\",\n",
    "    part=\"id,snippet\", maxResults=1\n",
    "  ).execute()\n",
    "\n",
    "  for search_result in  search_response.get(\"items\", []):\n",
    "      if search_result is not None:\n",
    "          video_id = search_result[\"id\"][\"videoId\"]\n",
    "          date_posted = search_result[\"snippet\"][\"publishedAt\"]\n",
    "          results = youtube.videos().list(\n",
    "                    part=\"statistics\", id=video_id\n",
    "                    ).execute()\n",
    "  return (video_id,\n",
    "          float(results[\"items\"][0][\"statistics\"][\"viewCount\"]),\n",
    "          float((datetime.now()-datetime.strptime(date_posted, \"%Y-%m-%dT%H:%M:%S.000Z\")).days))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('JXBFG2vsyCM', 5865358.0, 3402.0)\n"
     ]
    }
   ],
   "source": [
    "youtube_data = youtube_search(\"Nas Memory Lane\")\n",
    "print(youtube_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/JXBFG2vsyCM?rel=0&amp;controls=0&amp;showinfo=0\" frameborder=\"0\" allowfullscreen></iframe>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import HTML\n",
    "HTML('<iframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/JXBFG2vsyCM?rel=0&amp;controls=0&amp;showinfo=0\" frameborder=\"0\" allowfullscreen></iframe>')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* * *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Getting Definitions\n",
    "==================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from wiktionaryparser import WiktionaryParser\n",
    "\n",
    "def get_definition(word):\n",
    "    parser = WiktionaryParser()\n",
    "    worddef = parser.fetch(word)\n",
    "    \n",
    "    possible_defs = []\n",
    "    for entymologies in worddef:\n",
    "        for dd in entymologies[\"definitions\"]:\n",
    "            all_defs = re.sub(word+\"\\s*\\u200e\",\"\",\n",
    "                                  dd['text']).strip().split(\"\\n\")\n",
    "            all_gdefs = [d for d in all_defs if re.match(\"^\\(.*\\)$\",d) == None]\n",
    "            possible_defs.append((dd['partOfSpeech'],\n",
    "                                  all_gdefs))\n",
    "    return possible_defs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('noun',\n",
       "  ['(countable) A racquet: an implement with a handle connected to a round frame strung with wire, sinew, or plastic cords, and used to hit a ball, such as in tennis or a birdie in badminton.',\n",
       "   '(Canada) A snowshoe formed of cords stretched across a long and narrow frame of light wood.',\n",
       "   'A broad wooden shoe or patten for a man or horse, to allow walking on marshy or soft ground.']),\n",
       " ('verb', ['To strike with, or as if with, a racket.']),\n",
       " ('noun',\n",
       "  ['A loud noise.',\n",
       "   'A fraud or swindle; an illegal scheme for profit.',\n",
       "   '(dated, slang) A carouse; any reckless dissipation.',\n",
       "   '(dated, slang) Something taking place considered as exciting, trying, unusual, etc. or as an ordeal.'])]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_definition(\"racket\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('noun',\n",
       "  ['An English dessert made from a mixture of thick custard, fruit, sponge cake, jelly and whipped cream.',\n",
       "   'An insignificant amount.',\n",
       "   'Anything that is of little importance or worth.',\n",
       "   'A particular kind of pewter.',\n",
       "   '(uncountable) Utensils made from this particular kind of pewter.']),\n",
       " ('verb',\n",
       "  ['(intransitive) To deal with something as if it were of little importance or worth.',\n",
       "   '(intransitive) To act, speak, or otherwise behave with jest.',\n",
       "   '(intransitive) To inconsequentially toy with something.',\n",
       "   '(transitive) To squander or waste.'])]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_definition(\"trifle\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get Definitions from Context\n",
    "============================"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "word to christ, a disciple of streets, trifle on beats\n"
     ]
    }
   ],
   "source": [
    "lyrics = rap_data.loc[23375][\"lyrics\"]\n",
    "rap_sentence = [line for line in lyrics.split(\"\\n\") if word in line][0]\n",
    "print(rap_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('word', 'NOUN'), ('to', 'PRT'), ('christ', 'VERB'), (',', '.'), ('a', 'DET'), ('disciple', 'NOUN'), ('of', 'ADP'), ('streets', 'NOUN'), (',', '.'), ('trifle', 'NOUN'), ('on', 'ADP'), ('beats', 'NOUN')]\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "split_sentence = nltk.word_tokenize(rap_sentence)\n",
    "tagged_sentence = nltk.pos_tag(split_sentence,tagset=\"universal\")\n",
    "print(tagged_sentence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Universal Part of Speech Tags\n",
    "-----------------------------\n",
    "\n",
    "- VERB - verbs (all tenses and modes)\n",
    "- NOUN - nouns (common and proper)\n",
    "- PRON - pronouns \n",
    "- ADJ - adjectives\n",
    "- ADV - adverbs\n",
    "- ADP - adpositions (prepositions and postpositions)\n",
    "- CONJ - conjunctions\n",
    "- DET - determiners\n",
    "- NUM - cardinal numbers\n",
    "- PRT - particles or other function words\n",
    "- X - other: foreign words, typos, abbreviations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('word', 'NOUN'), ('to', 'PRT'), ('christ', 'VERB'), (',', '.'), ('a', 'DET'), ('disciple', 'NOUN'), ('of', 'ADP'), ('streets', 'NOUN'), (',', '.'), ('I', 'PRON'), ('trifle', 'VERB'), ('on', 'ADP'), ('beats', 'NOUN')]\n"
     ]
    }
   ],
   "source": [
    "rap_sentence = \"word to christ, a disciple of streets, I trifle on beats\"\n",
    "split_sentence = nltk.word_tokenize(rap_sentence)\n",
    "tagged_sentence = nltk.pos_tag(split_sentence,tagset=\"universal\")\n",
    "print(tagged_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_definition_with_sentence(word, rap_sentence):\n",
    "    split_sentence = nltk.word_tokenize(rap_sentence)\n",
    "    tagged_sentence = nltk.pos_tag(split_sentence,\n",
    "                                      tagset=\"universal\")\n",
    "    \n",
    "    index_of_word = split_sentence.index(word)\n",
    "    pos_of_word = tagged_sentence[index_of_word][1].lower()\n",
    "    \n",
    "    parser = WiktionaryParser()\n",
    "    worddef = parser.fetch(word)\n",
    "\n",
    "    possible_defs = []\n",
    "    for entymologies in worddef:\n",
    "        for dd in entymologies[\"definitions\"]:\n",
    "            part_of_speech = dd['partOfSpeech']\n",
    "            all_defs = re.sub(word+\"\\s*\\u200e\",\"\",\n",
    "                                  dd['text']).strip().split(\"\\n\")\n",
    "            all_gdefs = [d for d in all_defs if re.match(\"^\\(.*\\)$\",d) == None]\n",
    "            \n",
    "            # Take the first definition that matches part of speech\n",
    "            if part_of_speech == pos_of_word:\n",
    "                return (dd['partOfSpeech'], all_gdefs)\n",
    "\n",
    "    return (\"N/A\",\"N/A\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('verb',\n",
       " ['(intransitive) To deal with something as if it were of little importance or worth.',\n",
       "  '(intransitive) To act, speak, or otherwise behave with jest.',\n",
       "  '(intransitive) To inconsequentially toy with something.',\n",
       "  '(transitive) To squander or waste.'])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_definition_with_sentence(\"trifle\", rap_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_definition_with_lyrics(word, lyrics):\n",
    "    rap_sentence = [line for line in lyrics.split(\"\\n\") if word in line][0]\n",
    "    return get_definition_with_sentence(word, rap_sentence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* * *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract Surrounding Lines\n",
    "====================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lyrics_split=[\"Twinkle, twinkle, little star\",\n",
    "\"How I wonder what you are\",\n",
    "\"Up above the world so high\",\n",
    "\"Like a diamond in the sky\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_rhymegroup(target_word, lyrics_split):\n",
    "    group_index_of_target = -1\n",
    "    \n",
    "    for groupid, line in enumerate(lyrics_split):\n",
    "        if target_word in line:\n",
    "            group_index_of_target = groupid\n",
    "\n",
    "    if group_index_of_target > 0 and group_index_of_target < len(lyrics_split)-1:\n",
    "        return lyrics_split[group_index_of_target-1:group_index_of_target+2]\n",
    "    else:\n",
    "        return \"N/A\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['How I wonder what you are',\n",
       " 'Up above the world so high',\n",
       " 'Like a diamond in the sky']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_rhymegroup(\"high\", lyrics_split)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract Rhyming Couplet\n",
    "====================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pronouncing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['adar',\n",
       " 'afar',\n",
       " 'ahr',\n",
       " 'ajar',\n",
       " 'algar',\n",
       " 'all-star',\n",
       " 'allar',\n",
       " 'almanzar',\n",
       " 'almodovar',\n",
       " 'alvare',\n",
       " 'amar',\n",
       " 'andujar',\n",
       " 'aquilar',\n",
       " 'ar',\n",
       " 'are',\n",
       " 'avelar',\n",
       " 'azar',\n",
       " 'azhar',\n",
       " 'baar',\n",
       " 'babar']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pronouncing.rhymes(\"star\")[0:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['HH AY1']\n",
      "['S K AY1']\n"
     ]
    }
   ],
   "source": [
    "print(pronouncing.phones_for_word(\"high\"))\n",
    "print(pronouncing.phones_for_word(\"sky\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['AO1 R AH0 N JH', 'AO1 R IH0 N JH']\n",
      "['HH IH1 N JH']\n"
     ]
    }
   ],
   "source": [
    "print(pronouncing.phones_for_word(\"orange\"))\n",
    "print(pronouncing.phones_for_word(\"hinge\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rhymes_per_line(lyrics_split):\n",
    "    rhymes = []\n",
    "    for line in lyrics_split:\n",
    "        words = line.strip().split()\n",
    "        last_word = words[-1].strip('.,?!;:')\n",
    "        last_word_p = pronouncing.phones_for_word(last_word)\n",
    "        if len(last_word_p) > 0:\n",
    "            rhymes.append((pronouncing.rhyming_part(last_word_p[0]),line))\n",
    "    return rhymes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('AA1 R', 'Twinkle, twinkle, little star'),\n",
       " ('AA1 R', 'How I wonder what you are'),\n",
       " ('AY1', 'Up above the world so high'),\n",
       " ('AY1', 'Like a diamond in the sky')]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rhymes_per_line(lyrics_split)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* * *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['i rap for listeners, blunt heads, fly ladies and prisoners', 'henessey holders and old school niggaz, then i be dissin a', 'unofficial that smoke woolie thai', 'i dropped out of kooley high, gassed up by a cokehead cutie pie', \"jungle survivor, fuck who's the liver\", 'my man put the battery in my back, a differencem from energizer', 'sentence begins indented.. with formality', \"my duration's infinite, moneywise or physiology\", \"poetry, that's a part of me, retardedly bop\", 'i drop the ancient manifested hip-hop, straight off the block', 'i reminisce on park jams, my man was shot for his sheep coat', 'childhood lesson make me see him drop in my weed smoke', \"it's real, grew up in trife life, did times or white lines\", 'the hype vice, murderous nighttimes, and knife fights invite crimes', 'chill on the block with cog-nac, hold strap', \"with my peeps that's into drug money, market into rap\", 'no sign of the beast in the blue chrysler, i guess that means peace', 'for niggaz no sheisty vice to just snipe ya', 'start off the dice-rollin mats for craps to cee-lo ', 'with sidebets, i roll a deuce, nothin below (peace god!)', 'peace god -- now the shit is explained', \"i'm takin niggaz on a trip straight through memory lane\", \"it's like that y'all .. it's like that y'all .. it's like that y'all\", 'chorus: repeat scratches 4x', '\"now let me take a trip down memory lane\" -> bizmarkie', '\\t\"comin outta queensbridge\"', '[nas]', 'one for the money', 'two for pussy and foreign cars', 'three for alize niggaz deceased or behind bars', 'i rap divine gods check the prognosis, is it real or showbiz?', 'my window faces shootouts, drug overdoses', 'live amongst no roses, only the drama, for real', 'a nickel-plate is my fate, my medicine is the ganja']\n"
     ]
    }
   ],
   "source": [
    "lyrics_split = [line for line in df_data[\"lyrics\"][23375].split(\"\\n\") if len(line) > 0]\n",
    "print(lyrics_split[10:-30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "target_word = \"trifle\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_rhymes(lyrics_split):\n",
    "    lines_with_rhyming_parts = list()\n",
    "    for line in lyrics_split:\n",
    "        words = line.split()\n",
    "        last_word = words[-1].strip('.,?!;:') # .strip() to remove any punctuation\n",
    "        last_word_p = pronouncing.phones_for_word(last_word)\n",
    "        \n",
    "        if len(last_word_p) > 0:\n",
    "            if len(last_word_p) > 1:\n",
    "                last_word_p = [last_word_p[0],]\n",
    "                \n",
    "            for phones in last_word_p:\n",
    "                rhyming_part = pronouncing.rhyming_part(phones)\n",
    "                line_with_part = [rhyming_part[:2], line]\n",
    "                #print(line_with_part)\n",
    "                lines_with_rhyming_parts.append(line_with_part)\n",
    "        else:\n",
    "            line_with_part = [\"N/A\", line]\n",
    "            lines_with_rhyming_parts.append(line_with_part)\n",
    "            \n",
    "    return lines_with_rhyming_parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rhyme_lines = get_rhymes(lyrics_split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['OW', 'i rap divine gods check the prognosis, is it real or showbiz?'],\n",
       " ['OW', 'my window faces shootouts, drug overdoses'],\n",
       " ['IY', 'live amongst no roses, only the drama, for real'],\n",
       " ['N/A', 'a nickel-plate is my fate, my medicine is the ganja'],\n",
       " ['EY', \"here's my basis, my razor embraces, many faces\"],\n",
       " ['EY', 'your telephone blowin, black stitches or fat shoelaces'],\n",
       " ['OW', \"peoples are petrol, dramatic automatic fo'-fo' i let blow\"],\n",
       " ['OW', \"and back down po-po when i'm vexed so\"],\n",
       " ['AE', \"my pen taps the paper then my brain's blank\"],\n",
       " ['AE', 'i see dark streets, hustlin brothers who keep the same rank'],\n",
       " ['EY', 'pumpin for somethin, some uprise, plus some fail'],\n",
       " ['EY', 'judges hangin niggaz, uncorrect bails, for direct sales'],\n",
       " ['EY', 'my intellect prevails from a hangin cross with nails'],\n",
       " ['IY', \"i reinforce the frail, with lyrics that's real\"],\n",
       " ['IY', 'word to christ, a disciple of streets, trifle on beats'],\n",
       " ['IY', 'i decifer prophecies through a mic and say peace.'],\n",
       " ['N/A', 'i hung around the older crews while they sling smack to dingbats'],\n",
       " ['AE', \"they spoke of fat cat, that nigga's name made bell rings, black\"],\n",
       " ['IH', 'some fiends scream, about supreme team, a jamaica queens thing'],\n",
       " ['OW', 'uptown was alpo, son, heard he was kingpin, yo']]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rhyme_lines[40:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "import operator\n",
    "\n",
    "grouped_rhymes = []\n",
    "for key,group in itertools.groupby(rhyme_lines, operator.itemgetter(0)):\n",
    "    merged_group = [g[1] for g in group]\n",
    "    grouped_rhymes.append(list(merged_group))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['chill on the block with cog-nac, hold strap',\n",
       "  \"with my peeps that's into drug money, market into rap\"],\n",
       " ['no sign of the beast in the blue chrysler, i guess that means peace'],\n",
       " ['for niggaz no sheisty vice to just snipe ya'],\n",
       " ['start off the dice-rollin mats for craps to cee-lo ',\n",
       "  'with sidebets, i roll a deuce, nothin below (peace god!)'],\n",
       " ['peace god -- now the shit is explained',\n",
       "  \"i'm takin niggaz on a trip straight through memory lane\"],\n",
       " [\"it's like that y'all .. it's like that y'all .. it's like that y'all\"],\n",
       " ['chorus: repeat scratches 4x',\n",
       "  '\"now let me take a trip down memory lane\" -> bizmarkie',\n",
       "  '\\t\"comin outta queensbridge\"',\n",
       "  '[nas]'],\n",
       " ['one for the money'],\n",
       " ['two for pussy and foreign cars',\n",
       "  'three for alize niggaz deceased or behind bars'],\n",
       " ['i rap divine gods check the prognosis, is it real or showbiz?',\n",
       "  'my window faces shootouts, drug overdoses'],\n",
       " ['live amongst no roses, only the drama, for real'],\n",
       " ['a nickel-plate is my fate, my medicine is the ganja'],\n",
       " [\"here's my basis, my razor embraces, many faces\",\n",
       "  'your telephone blowin, black stitches or fat shoelaces'],\n",
       " [\"peoples are petrol, dramatic automatic fo'-fo' i let blow\",\n",
       "  \"and back down po-po when i'm vexed so\"],\n",
       " [\"my pen taps the paper then my brain's blank\",\n",
       "  'i see dark streets, hustlin brothers who keep the same rank'],\n",
       " ['pumpin for somethin, some uprise, plus some fail',\n",
       "  'judges hangin niggaz, uncorrect bails, for direct sales',\n",
       "  'my intellect prevails from a hangin cross with nails'],\n",
       " [\"i reinforce the frail, with lyrics that's real\",\n",
       "  'word to christ, a disciple of streets, trifle on beats',\n",
       "  'i decifer prophecies through a mic and say peace.'],\n",
       " ['i hung around the older crews while they sling smack to dingbats'],\n",
       " [\"they spoke of fat cat, that nigga's name made bell rings, black\"],\n",
       " ['some fiends scream, about supreme team, a jamaica queens thing']]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grouped_rhymes[15:35]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rhymegroup(target_word, grouped_rhymes):\n",
    "    group_index_of_target = -1\n",
    "    \n",
    "    for groupid, rhymes in enumerate(grouped_rhymes):\n",
    "        #print(groupid, rhymes)\n",
    "        for line in rhymes:\n",
    "            if target_word in line:\n",
    "                group_index_of_target = groupid\n",
    "    \n",
    "    if group_index_of_target != -1:\n",
    "        return grouped_rhymes[group_index_of_target]\n",
    "    else:\n",
    "        return \"N/A\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"i reinforce the frail, with lyrics that's real\",\n",
       " 'word to christ, a disciple of streets, trifle on beats',\n",
       " 'i decifer prophecies through a mic and say peace.']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rhymegroup(\"trifle\", grouped_rhymes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rhymegroup_from_word(word, lyrics):\n",
    "    lyrics_split = [line for line in lyrics.split(\"\\n\") if len(line) > 0]\n",
    "    grouped_rhymes = []\n",
    "    for key,group in itertools.groupby(rhyme_lines, operator.itemgetter(0)):\n",
    "        merged_group = [g[1] for g in group]\n",
    "        grouped_rhymes.append(list(merged_group))\n",
    "    return rhymegroup(word, grouped_rhymes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* * *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Posting on Tumblr\n",
    "================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false,
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "# Get API key, https://www.tumblr.com/docs/en/api/v2 and do OATHv1\n",
    "tumblr_client = ''\n",
    "tumblr_secret = ''\n",
    "access_key    = ''\n",
    "access_secret = ''\n",
    "\n",
    "import pytumblr\n",
    "\n",
    "user = pytumblr.TumblrRestClient(\n",
    "    tumblr_client, tumblr_secret,\n",
    "    access_key, access_secret)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def post_template(word, part_of_speech, worddef, lyrics, artist, song, youtube=None):\n",
    "    \n",
    "    post = '''<p><a href=\"http://en.wiktionary.org/wiki/{}\">{}</a> '''.format(word, word)\n",
    "    post += '''- {} -\\xa0 {}</p>'''.format(part_of_speech, worddef)\n",
    "\n",
    "    post += \"<p>\"\n",
    "    for line in lyrics:\n",
    "        if word in line:\n",
    "            post += line.replace(word,\"<b>\"+word+\"</b>\")+\"<br />\"\n",
    "        else:\n",
    "            post += line+\"<br />\"\n",
    "    post += \"</p>\"\n",
    "    \n",
    "    if youtube is not None:\n",
    "        post += '''<p>-{} on\\xa0“'''.format(artist)\n",
    "        post += '''<a href=\"https://www.youtube.com/watch?v={}\">{}</a>”</p>'''.format(youtube,\n",
    "                                                                                      song)\n",
    "    else:\n",
    "        post += '''<p>-{} on\\xa0“{}”</p>'''.format(artist,song)        \n",
    "    \n",
    "    return post"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Puttin' it Together\n",
    "==================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "artist                                                       Nas\n",
      "filename       Lyrics/ohhla.com/anonymous/nas/illmatic/memory...\n",
      "lyrics         \\t(check that shit)\\n\\taight fuck that shit, w...\n",
      "matches        {dingbats, reminisce, gassed, takin, trifle, s...\n",
      "rap_matches                               {dingbats, physiology}\n",
      "song                            Memory Lane (Sittin' in Da Park)\n",
      "Name: 23375, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(rap_data.loc[23375])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "word = \"trifle\"\n",
    "artist = rap_data.loc[23375].artist\n",
    "song = rap_data.loc[23375].song\n",
    "\n",
    "part_of_speech, worddef = get_definition_with_lyrics(word,\n",
    "                                                       rap_data.loc[23375].lyrics)\n",
    "lyrics = rhymegroup_from_word(word, rap_data.loc[23375].lyrics)\n",
    "youtube = youtube_search(artist + \" \" + song)[0]\n",
    "\n",
    "slug = word+\"-\"+part_of_speech+\"-\"+artist\n",
    "\n",
    "#print(word, part_of_speech, worddef[0], lyrics, artist, song, youtube)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "post_body = post_template(word, part_of_speech, worddef[0], lyrics, artist, song, youtube)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 153129975615}"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user.create_text(\"rapwords\",\n",
    "                 format=\"html\",\n",
    "                 state=\"published\",\n",
    "                 slug=slug,\n",
    "                 body=post_body,\n",
    "                 tags=[part_of_speech],)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Peace\n",
    "====="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Rap Trifle](assets/RapTrifle.png)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}