Chapter 13 - Textual Data Analysis with NLKT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NLTK Downloader\n",
      "---------------------------------------------------------------------------\n",
      "    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n",
      "---------------------------------------------------------------------------\n",
      "Downloader> l\n",
      "\n",
      "Packages:\n",
      "  [ ] abc................. Australian Broadcasting Commission 2006\n",
      "  [ ] alpino.............. Alpino Dutch Treebank\n",
      "  [*] averaged_perceptron_tagger Averaged Perceptron Tagger\n",
      "  [ ] averaged_perceptron_tagger_ru Averaged Perceptron Tagger (Russian)\n",
      "  [ ] basque_grammars..... Grammars for Basque\n",
      "  [ ] biocreative_ppi..... BioCreAtIvE (Critical Assessment of Information\n",
      "                           Extraction Systems in Biology)\n",
      "  [ ] bllip_wsj_no_aux.... BLLIP Parser: WSJ Model\n",
      "  [ ] book_grammars....... Grammars from NLTK Book\n",
      "  [ ] brown............... Brown Corpus\n",
      "  [ ] brown_tei........... Brown Corpus (TEI XML Version)\n",
      "  [ ] cess_cat............ CESS-CAT Treebank\n",
      "  [ ] cess_esp............ CESS-ESP Treebank\n",
      "  [ ] chat80.............. Chat-80 Data Files\n",
      "  [ ] city_database....... City Database\n",
      "  [ ] cmudict............. The Carnegie Mellon Pronouncing Dictionary (0.6)\n",
      "  [ ] comparative_sentences Comparative Sentence Dataset\n",
      "  [ ] comtrans............ ComTrans Corpus Sample\n",
      "  [ ] conll2000........... CONLL 2000 Chunking Corpus\n",
      "  [ ] conll2002........... CONLL 2002 Named Entity Recognition Corpus\n",
      "Hit Enter to continue: \n",
      "  [ ] conll2007........... Dependency Treebanks from CoNLL 2007 (Catalan\n",
      "                           and Basque Subset)\n",
      "  [ ] crubadan............ Crubadan Corpus\n",
      "  [ ] dependency_treebank. Dependency Parsed Treebank\n",
      "  [ ] dolch............... Dolch Word List\n",
      "  [ ] europarl_raw........ Sample European Parliament Proceedings Parallel\n",
      "                           Corpus\n",
      "  [ ] floresta............ Portuguese Treebank\n",
      "  [ ] framenet_v15........ FrameNet 1.5\n",
      "  [ ] framenet_v17........ FrameNet 1.7\n",
      "  [ ] gazetteers.......... Gazeteer Lists\n",
      "  [ ] genesis............. Genesis Corpus\n",
      "  [*] gutenberg........... Project Gutenberg Selections\n",
      "  [ ] ieer................ NIST IE-ER DATA SAMPLE\n",
      "  [ ] inaugural........... C-Span Inaugural Address Corpus\n",
      "  [ ] indian.............. Indian Language POS-Tagged Corpus\n",
      "  [ ] jeita............... JEITA Public Morphologically Tagged Corpus (in\n",
      "                           ChaSen format)\n",
      "  [ ] kimmo............... PC-KIMMO Data Files\n",
      "  [ ] knbc................ KNB Corpus (Annotated blog corpus)\n",
      "  [ ] large_grammars...... Large context-free and feature-based grammars\n",
      "                           for parser comparison\n",
      "Hit Enter to continue: \n",
      "  [ ] lin_thesaurus....... Lin's Dependency Thesaurus\n",
      "  [ ] mac_morpho.......... MAC-MORPHO: Brazilian Portuguese news text with\n",
      "                           part-of-speech tags\n",
      "  [ ] machado............. Machado de Assis -- Obra Completa\n",
      "  [ ] masc_tagged......... MASC Tagged Corpus\n",
      "  [ ] maxent_ne_chunker... ACE Named Entity Chunker (Maximum entropy)\n",
      "  [ ] maxent_treebank_pos_tagger Treebank Part of Speech Tagger (Maximum entropy)\n",
      "  [ ] moses_sample........ Moses Sample Models\n",
      "  [*] movie_reviews....... Sentiment Polarity Dataset Version 2.0\n",
      "  [ ] mte_teip5........... MULTEXT-East 1984 annotated corpus 4.0\n",
      "  [ ] mwa_ppdb............ The monolingual word aligner (Sultan et al.\n",
      "                           2015) subset of the Paraphrase Database.\n",
      "  [*] names............... Names Corpus, Version 1.3 (1994-03-29)\n",
      "  [ ] nombank.1.0......... NomBank Corpus 1.0\n",
      "  [ ] nonbreaking_prefixes Non-Breaking Prefixes (Moses Decoder)\n",
      "  [ ] nps_chat............ NPS Chat\n",
      "  [ ] omw................. Open Multilingual Wordnet\n",
      "  [ ] opinion_lexicon..... Opinion Lexicon\n",
      "  [ ] panlex_swadesh...... PanLex Swadesh Corpora\n",
      "  [ ] paradigms........... Paradigm Corpus\n",
      "  [ ] pe08................ Cross-Framework and Cross-Domain Parser\n",
      "                           Evaluation Shared Task\n",
      "Hit Enter to continue: \n",
      "  [ ] perluniprops........ perluniprops: Index of Unicode Version 7.0.0\n",
      "                           character properties in Perl\n",
      "  [ ] pil................. The Patient Information Leaflet (PIL) Corpus\n",
      "  [ ] pl196x.............. Polish language of the XX century sixties\n",
      "  [ ] porter_test......... Porter Stemmer Test Files\n",
      "  [ ] ppattach............ Prepositional Phrase Attachment Corpus\n",
      "  [ ] problem_reports..... Problem Report Corpus\n",
      "  [ ] product_reviews_1... Product Reviews (5 Products)\n",
      "  [ ] product_reviews_2... Product Reviews (9 Products)\n",
      "  [ ] propbank............ Proposition Bank Corpus 1.0\n",
      "  [ ] pros_cons........... Pros and Cons\n",
      "  [ ] ptb................. Penn Treebank\n",
      "  [*] punkt............... Punkt Tokenizer Models\n",
      "  [ ] qc.................. Experimental Data for Question Classification\n",
      "  [ ] reuters............. The Reuters-21578 benchmark corpus, ApteMod\n",
      "                           version\n",
      "  [ ] rslp................ RSLP Stemmer (Removedor de Sufixos da Lingua\n",
      "                           Portuguesa)\n",
      "  [ ] rte................. PASCAL RTE Challenges 1, 2, and 3\n",
      "  [ ] sample_grammars..... Sample Grammars\n",
      "  [ ] semcor.............. SemCor 3.0\n",
      "Hit Enter to continue: \n",
      "  [ ] senseval............ SENSEVAL 2 Corpus: Sense Tagged Text\n",
      "  [ ] sentence_polarity... Sentence Polarity Dataset v1.0\n",
      "  [ ] sentiwordnet........ SentiWordNet\n",
      "  [ ] shakespeare......... Shakespeare XML Corpus Sample\n",
      "  [ ] sinica_treebank..... Sinica Treebank Corpus Sample\n",
      "  [ ] smultron............ SMULTRON Corpus Sample\n",
      "  [ ] snowball_data....... Snowball Data\n",
      "  [ ] spanish_grammars.... Grammars for Spanish\n",
      "  [ ] state_union......... C-Span State of the Union Address Corpus\n",
      "  [*] stopwords........... Stopwords Corpus\n",
      "  [ ] subjectivity........ Subjectivity Dataset v1.0\n",
      "  [ ] swadesh............. Swadesh Wordlists\n",
      "  [ ] switchboard......... Switchboard Corpus Sample\n",
      "  [ ] tagsets............. Help on Tagsets\n",
      "  [ ] timit............... TIMIT Corpus Sample\n",
      "  [ ] toolbox............. Toolbox Sample Files\n",
      "  [ ] treebank............ Penn Treebank Sample\n",
      "  [ ] twitter_samples..... Twitter Samples\n",
      "  [ ] udhr2............... Universal Declaration of Human Rights Corpus\n",
      "                           (Unicode Version)\n",
      "  [ ] udhr................ Universal Declaration of Human Rights Corpus\n",
      "Hit Enter to continue: \n",
      "  [ ] unicode_samples..... Unicode Samples\n",
      "  [ ] universal_tagset.... Mappings to the Universal Part-of-Speech Tagset\n",
      "  [ ] universal_treebanks_v20 Universal Treebanks Version 2.0\n",
      "  [ ] vader_lexicon....... VADER Sentiment Lexicon\n",
      "  [ ] verbnet............. VerbNet Lexicon, Version 2.1\n",
      "  [ ] webtext............. Web Text Corpus\n",
      "  [ ] wmt15_eval.......... Evaluation data from WMT15\n",
      "  [ ] word2vec_sample..... Word2Vec Sample\n",
      "  [ ] wordnet............. WordNet\n",
      "  [ ] wordnet_ic.......... WordNet-InfoContent\n",
      "  [ ] words............... Word Lists\n",
      "  [ ] ycoe................ York-Toronto-Helsinki Parsed Corpus of Old\n",
      "                           English Prose\n",
      "\n",
      "Collections:\n",
      "  [P] all-corpora......... All the corpora\n",
      "  [P] all-nltk............ All packages available on nltk_data gh-pages\n",
      "                           branch\n",
      "  [P] all................. All packages\n",
      "  [P] book................ Everything used in the NLTK Book\n",
      "  [P] popular............. Popular packages\n",
      "  [P] tests............... Packages for running tests\n",
      "Hit Enter to continue: \n",
      "  [ ] third-party......... Third-party data packages\n",
      "\n",
      "([*] marks installed packages; [P] marks partially installed collections)\n",
      "\n",
      "---------------------------------------------------------------------------\n",
      "    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n",
      "---------------------------------------------------------------------------\n",
      "Downloader> d\n",
      "\n",
      "Download which package (l=list; x=cancel)?\n",
      "  Identifier> gutenberg\n",
      "    Downloading package gutenberg to\n",
      "        C:\\Users\\nelli\\AppData\\Roaming\\nltk_data...\n",
      "      Package gutenberg is already up-to-date!\n",
      "\n",
      "---------------------------------------------------------------------------\n",
      "    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n",
      "---------------------------------------------------------------------------\n",
      "Downloader> q\n"
     ]
    }
   ],
   "source": [
    "nltk.download_shell()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gutenberg files :  ['austen-emma.txt', 'austen-persuasion.txt', 'austen-sense.txt', 'bible-kjv.txt', 'blake-poems.txt', 'bryant-stories.txt', 'burgess-busterbrown.txt', 'carroll-alice.txt', 'chesterton-ball.txt', 'chesterton-brown.txt', 'chesterton-thursday.txt', 'edgeworth-parents.txt', 'melville-moby_dick.txt', 'milton-paradise.txt', 'shakespeare-caesar.txt', 'shakespeare-hamlet.txt', 'shakespeare-macbeth.txt', 'whitman-leaves.txt']\n"
     ]
    }
   ],
   "source": [
    "gb = nltk.corpus.gutenberg \n",
    "print(\"Gutenberg files : \", gb.fileids()) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "macbeth = nltk.corpus.gutenberg.words('shakespeare-macbeth.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23140"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(macbeth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[',\n",
       " 'The',\n",
       " 'Tragedie',\n",
       " 'of',\n",
       " 'Macbeth',\n",
       " 'by',\n",
       " 'William',\n",
       " 'Shakespeare',\n",
       " '1603',\n",
       " ']']"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "macbeth[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['[',\n",
       "  'The',\n",
       "  'Tragedie',\n",
       "  'of',\n",
       "  'Macbeth',\n",
       "  'by',\n",
       "  'William',\n",
       "  'Shakespeare',\n",
       "  '1603',\n",
       "  ']'],\n",
       " ['Actus', 'Primus', '.'],\n",
       " ['Scoena', 'Prima', '.'],\n",
       " ['Thunder', 'and', 'Lightning', '.'],\n",
       " ['Enter', 'three', 'Witches', '.']]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "macbeth_sents = nltk.corpus.gutenberg.sents('shakespeare-macbeth.txt')\n",
    "macbeth_sents[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[',\n",
       " 'The',\n",
       " 'Tragedie',\n",
       " 'of',\n",
       " 'Macbeth',\n",
       " 'by',\n",
       " 'William',\n",
       " 'Shakespeare',\n",
       " '1603',\n",
       " ']']"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = nltk.Text(macbeth)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 3 of 3 matches:\n",
      "nts with Dishes and Seruice ouer the Stage . Then enter Macbeth Macb . If it we\n",
      "with mans Act , Threatens his bloody Stage : byth ' Clock ' tis Day , And yet d\n",
      " struts and frets his houre vpon the Stage , And then is heard no more . It is \n"
     ]
    }
   ],
   "source": [
    "text.concordance('Stage')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fogge ayre bleeding reuolt good shew heeles skie other sea feare\n",
      "consequence heart braine seruice herbenger lady round deed doore\n"
     ]
    }
   ],
   "source": [
    "text.similar('Stage')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the_then bloody_byth the_and\n"
     ]
    }
   ],
   "source": [
    "text.common_contexts(['Stage'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Frequency distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(',', 1962),\n",
       " ('.', 1235),\n",
       " (\"'\", 637),\n",
       " ('the', 531),\n",
       " (':', 477),\n",
       " ('and', 376),\n",
       " ('I', 333),\n",
       " ('of', 315),\n",
       " ('to', 311),\n",
       " ('?', 241)]"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fd = nltk.FreqDist(macbeth) \n",
    "fd.most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\nelli\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "179\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['once',\n",
       " 'o',\n",
       " \"hasn't\",\n",
       " 'i',\n",
       " \"you'd\",\n",
       " 'because',\n",
       " 'yourselves',\n",
       " 'being',\n",
       " \"wouldn't\",\n",
       " 'been']"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sw = set(nltk.corpus.stopwords.words('english')) \n",
    "print(len(sw))\n",
    "list(sw)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "macbeth_filtered = [w for w in macbeth if w.lower() not in sw] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14946"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(macbeth_filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(',', 1962),\n",
       " ('.', 1235),\n",
       " (\"'\", 637),\n",
       " (':', 477),\n",
       " ('?', 241),\n",
       " ('Macb', 137),\n",
       " ('haue', 117),\n",
       " ('-', 100),\n",
       " ('Enter', 80),\n",
       " ('thou', 63)]"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fd = nltk.FreqDist(macbeth_filtered) \n",
    "fd.most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "punctuation = set(string.punctuation) \n",
    "macbeth_filtered2 = [w.lower() for w in macbeth if w.lower() not in sw and w.lower() not in punctuation] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('macb', 137),\n",
       " ('haue', 122),\n",
       " ('thou', 90),\n",
       " ('enter', 81),\n",
       " ('shall', 68),\n",
       " ('macbeth', 62),\n",
       " ('vpon', 62),\n",
       " ('thee', 61),\n",
       " ('macd', 58),\n",
       " ('vs', 57)]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fd = nltk.FreqDist(macbeth_filtered2) \n",
    "fd.most_common(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Una selezione raffinata delle parole lunghe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "long_words = [w for w in macbeth if len(w) > 12]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Assassination',\n",
       " 'Chamberlaines',\n",
       " 'Distinguishes',\n",
       " 'Gallowgrosses',\n",
       " 'Metaphysicall',\n",
       " 'Northumberland',\n",
       " 'Voluptuousnesse',\n",
       " 'commendations',\n",
       " 'multitudinous',\n",
       " 'supernaturall',\n",
       " 'vnaccompanied']"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(long_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Auaricious',\n",
       " 'Gracious',\n",
       " 'Industrious',\n",
       " 'Iudicious',\n",
       " 'Luxurious',\n",
       " 'Malicious',\n",
       " 'Obliuious',\n",
       " 'Pious',\n",
       " 'Rebellious',\n",
       " 'compunctious',\n",
       " 'furious',\n",
       " 'gracious',\n",
       " 'pernicious',\n",
       " 'pernitious',\n",
       " 'pious',\n",
       " 'precious',\n",
       " 'rebellious',\n",
       " 'sacrilegious',\n",
       " 'serious',\n",
       " 'spacious',\n",
       " 'tedious']"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ious_words = [w for w in macbeth if 'ious' in w]\n",
    "ious_words = set(ious_words)\n",
    "sorted(ious_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('enter', 'macbeth'), 16),\n",
       " (('exeunt', 'scena'), 15),\n",
       " (('thane', 'cawdor'), 13),\n",
       " (('knock', 'knock'), 10),\n",
       " (('st', 'thou'), 9),\n",
       " (('thou', 'art'), 9),\n",
       " (('lord', 'macb'), 9),\n",
       " (('haue', 'done'), 8),\n",
       " (('macb', 'haue'), 8),\n",
       " (('good', 'lord'), 8),\n",
       " (('let', 'vs'), 7),\n",
       " (('enter', 'lady'), 7),\n",
       " (('wee', 'l'), 7),\n",
       " (('would', 'st'), 6),\n",
       " (('macbeth', 'macb'), 6)]"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bgrms = nltk.FreqDist(nltk.bigrams(macbeth_filtered2)) \n",
    "bgrms.most_common(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('knock', 'knock', 'knock'), 6),\n",
       " (('enter', 'macbeth', 'macb'), 5),\n",
       " (('enter', 'three', 'witches'), 4),\n",
       " (('exeunt', 'scena', 'secunda'), 4),\n",
       " (('good', 'lord', 'macb'), 4),\n",
       " (('three', 'witches', '1'), 3),\n",
       " (('exeunt', 'scena', 'tertia'), 3),\n",
       " (('thunder', 'enter', 'three'), 3),\n",
       " (('exeunt', 'scena', 'quarta'), 3),\n",
       " (('scena', 'prima', 'enter'), 3)]"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tgrms = nltk.FreqDist(nltk.trigrams(macbeth_filtered2)) \n",
    "tgrms.most_common(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Accessing Text from the Web and from Disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from urllib import request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"http://www.gutenberg.org/files/2554/2554-0.txt\"\n",
    "response = request.urlopen(url)\n",
    "raw = response.read().decode('utf-8-sig')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1176964"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The Project Gutenberg EBook of Crime and Punishment, by Fyodor Dostoevsky\\r\\n'"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw[:75]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = nltk.word_tokenize(raw)\n",
    "webtext = nltk.Text(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The',\n",
       " 'Project',\n",
       " 'Gutenberg',\n",
       " 'EBook',\n",
       " 'of',\n",
       " 'Crime',\n",
       " 'and',\n",
       " 'Punishment',\n",
       " ',',\n",
       " 'by',\n",
       " 'Fyodor',\n",
       " 'Dostoevsky']"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "webtext[:12]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Working with HTML pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<!doctype html public \"-//W3C//DTD HTML 4.0 Transitional//EN\" \"http://www.w3.org/TR/REC-html40/loose.dtd\">\\r\\n<html>\\r\\n<hea'"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = \"http://news.bbc.co.uk/2/hi/health/2284783.stm\"\n",
    "html = request.urlopen(url).read().decode('utf8')\n",
    "html[:120]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "raw = BeautifulSoup(html,\"lxml\").get_text()\n",
    "tokens = nltk.word_tokenize(raw)\n",
    "text = nltk.Text(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Text: BBC NEWS | Health | Blondes 'to die...>"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 7 of 7 matches:\n",
      "they say too few people now carry the gene for blondes to last beyond the next t\n",
      " blonde hair is caused by a recessive gene . In order for a child to have blonde\n",
      "o have blonde hair , it must have the gene on both sides of the family in the gr\n",
      "here is a disadvantage of having that gene or by chance . They do n't disappear \n",
      "ndes would disappear is if having the gene was a disadvantage and I do not think\n",
      "mer's Polio campaign launched in Iraq Gene defect explains high blood pressure B\n",
      "mer's Polio campaign launched in Iraq Gene defect explains high blood pressure B\n"
     ]
    }
   ],
   "source": [
    "text.concordance('gene')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Movie Reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package movie_reviews to\n",
      "[nltk_data]     C:\\Users\\nelli\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package movie_reviews is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('movie_reviews')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "reviews = nltk.corpus.movie_reviews\n",
    "documents = [(list(reviews.words(fileid)), category)\n",
    "          for category in reviews.categories()\n",
    "          for fileid in reviews.fileids(category)]\n",
    "random.shuffle(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "any remake of an alfred hitchcock film is at best an uncertain project , as a perfect murder illustrates . frankly , dial m for murder is not one of the master director ' s greatest efforts , so there is ample room for improvement . unfortunately , instead of updating the script , ironing out some of the faults , and speeding up the pace a little , a perfect murder has inexplicably managed to eliminate almost everything that was worthwhile about dial m for murder , leaving behind the nearly - unwatchable wreckage of a would - be ' 90s thriller . almost all suspense films are loaded with plot implausibilities . the best thrillers keep viewers involved enough in what ' s going on so that these flaws in logic don ' t become apparent until long after the final credits have rolled . unfortunately , in a perfect murder , the faults are often so overt that we become aware of them as they ' re happening . this is a very bad sign . not only do such occurrences shatter any suspension of disbelief , but they have the astute viewer looking for the next such blunder . of course , in the case of a perfect murder , at least that gives an audience member something to do besides concentrating on the inane plot and the lifeless , cardboard characters . a perfect murder isn ' t a strict remake of dial m for murder , but it does borrow heavily from frederick knott ' s play ( which was also the source material for hitchcock ' s version , as well as a 1981 made - for - tv retelling ) . emily hayes ( gwyneth paltrow ) is the wealthy wife of powerful wall street mover - and - shaker steven hayes ( michael douglas ) . their marriage isn ' t going well -- emily resents steven ' s controlling instincts , and , as a form of rebellion , she is having an affair with a penniless painter , david shaw ( viggo mortensen ) . when steven learns of the relationship , he decides to confront david , but his approach isn ' t that of a typical cuckolded husband . instead of yelling or threatening , steven offers david a proposal that ' s too good to resist : for $ 500 , 000 in cash ( $ 100 , 000 before , the rest after ) , he is to break into steven ' s new york apartment and kill emily . ( of course , after getting the first payment , david never bothers to ask how he ' s supposed to get the rest . ) ultimately , i ' m not sure which of the three main characters we ' re supposed to be sympathetic to : the cold - hearted husband , who wants his wife dead so he can get his hands on her fortune ; her mercenary lover , who is willing to do the deed for half - a - million ; or the woman , who is happily carrying on an extramarital affair . not only are these individuals all profoundly dislikable , but they ' re not interesting . ( it ' s possible to make a good movie with detestable characters -- see reservoir dogs -- but there has to be something compelling about them , which , in this case , there isn ' t . ) steven , emily , and david are all lifted directly from the screenwriting 101 text book on stereotypes . the actors in this film are obviously just on hand to get their paychecks . michael douglas is playing the kind of heartless tycoon that he can do in his sleep -- he ' s gordon gekko with an unfaithful wife . gwyneth paltrow , who was recently delightful and appealing in sliding doors , is simply awful here . she now has the dubious distinction of have starred in two of 1998 ' s worst thrillers ( the other being hush ) . at least viggo mortensen ( g . i . jane ) has a little fun with his part , but then he usually does interesting things even in bad movies . the thin supporting cast includes david suchet , the star of \" poirot , \" as a police inspector , and sarita choudhury ( kama sutra ) as emily ' s best friend . a perfect murder is a plodding production that generates almost no suspense from beginning to end . there aren ' t many twists and turns in the unexpectedly linear script , which makes the ending inevitable almost from the start . it ' s surprising to see director andrew davis , the man behind the fugitive , involved in this mess , but , like his stars , he too needs to earn a living . it ' s just that remaking hitchcock , and doing it so badly , hardly seems to be an honorable way to go about getting the dough .\n"
     ]
    }
   ],
   "source": [
    "first_review = ' '.join(documents[0][0])\n",
    "print(first_review)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'neg'"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_words = nltk.FreqDist(w.lower() for w in reviews.words())\n",
    "word_features = list(all_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "def document_features(document, word_features): \n",
    "    document_words = set(document)\n",
    "    features = {}\n",
    "    for word in word_features:\n",
    "        features['{}'.format(word)] = (word in document_words)\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2000"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "featuresets = [(document_features(d,word_features), c) for (d,c) in documents]\n",
    "len(featuresets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set, est_set = featuresets[1500:], featuresets[:500]\n",
    "classifier = nltk.NaiveBayesClassifier.train(train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.88\n"
     ]
    }
   ],
   "source": [
    "print(nltk.classify.accuracy(classifier, test_set)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "                   blame = True              neg : pos    =     12.7 : 1.0\n",
      "             wonderfully = True              pos : neg    =     12.4 : 1.0\n",
      "                     era = True              pos : neg    =      9.5 : 1.0\n",
      "                   inept = True              neg : pos    =      8.1 : 1.0\n",
      "                   anger = True              pos : neg    =      7.8 : 1.0\n",
      "                    porn = True              neg : pos    =      7.3 : 1.0\n",
      "                  finest = True              pos : neg    =      7.2 : 1.0\n",
      "               assistant = True              pos : neg    =      7.2 : 1.0\n",
      "                    mass = True              pos : neg    =      6.6 : 1.0\n",
      "                realizes = True              pos : neg    =      6.6 : 1.0\n"
     ]
    }
   ],
   "source": [
    "classifier.show_most_informative_features(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    " \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}