Score2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    _create_unverified_https_context = ssl._create_unverified_context\n",
    "except AttributeError:\n",
    "    pass\n",
    "else:\n",
    "    ssl._create_default_https_context = _create_unverified_https_context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/scottcrawshaw/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "import ssl\n",
    "import heapq\n",
    "import re\n",
    "pd.options.mode.chained_assignment = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "training = pd.read_json('Grocery_and_Gourmet_Food_Reviews_training.json', lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "textOnly = training[['overall', 'reviewText']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "textOnly.dropna();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "textOnlyRatings = textOnly['overall']\n",
    "del textOnly['overall']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# textOnly is a dataframe with only text of review\n",
    "# textOnlyRatings is a dataframe with only rating of review.\n",
    "# Each index in textOnly corresponds to the same index in textOnlyRatings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([\"Can't wait to get cooking with these. I have heard about capers and had never tasted them. However aster tasting them I can't wait to get experimenting with some different recipes.\",\n",
       "       'Large and good tasting', 'wonderful', ...,\n",
       "       \"Quinine bitterness not the least bit detectable. In fact not bitter even when drunk straight up.. Overwhelming sourness from all the citric acid. May make a decent albeit pricey whiskey sour. Will neither buy again nor recommend it to anyone. Grocery store tonics are far far better at a small fraction of the price. C'est la vie!\",\n",
       "       \"i guess i'm so used to regular tonic, this did not impress me that much, even though it gets great reviews. thought i'd try it at least once.\",\n",
       "       \"Maybe this works when added to a drink, but I wanted to use it to make tonic water and the result tasted so bad when I added it to some soda water I made on my Sodastream that I couldn't drink it!  The remainder of the bottle of concentrate went down the drain.  Now I'm back to buying high-fructose-sweetened Schweppes and mixing it 1:1 with soda water to get something with reasonable flavor and not too sweet.\"],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textOnly.values.flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = textOnly.values.flatten()[:100] \n",
    "for i in range(len(dataset)): \n",
    "    dataset[i] = str(dataset[i]).lower() \n",
    "    dataset[i] = re.sub(r'\\W', ' ', dataset[i]) \n",
    "    dataset[i] = re.sub(r'\\s+', ' ', dataset[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['can t wait to get cooking with these i have heard about capers and had never tasted them however aster tasting them i can t wait to get experimenting with some different recipes '\n",
      " 'large and good tasting' 'wonderful' 'capers '\n",
      " 'these are good quality and cheaper by the use than those tiny bottles that they sell in the stores they are great in tuna sandwiches i like them in potato salad also they came very nicely packaged to prevent breakage i will buy them again '\n",
      " 'great purchase ' 'fantastic product perfect size and flavor '\n",
      " 'product as described thank you'\n",
      " 'not a capers expert but these are delish we always rinse them under cold water in a strainer first before adding to food to wash off excess salt just wish paesana would stop with the neverending price creep or maybe it s not them but amazon '\n",
      " 'look good'\n",
      " 'these are so salty that even with repeated changing of the liquid i have trouble with the taste in anything i use them for'\n",
      " 'these capers are so good i use them very often in couscous potato salad with lox and many other dishes simply cannot find 32 oz size jars in any stores so found these on amazon i ll keep ordering them '\n",
      " 'received quickly and exactly as adertised'\n",
      " 'super salty like sickeningly puckeringly salty i changed the brine out with fresh water and they are still too salty the new brine is just as salty as the original brine i don t know how but it is there is almost one gram of salt in a 4 gram serving of these capers that means 1 4 of this entire bottle is made up of salt '\n",
      " 'well packaged and tasty capers we have had several helpings in various dishes since it arrived the taste is still very good we are glad to have a large jar '\n",
      " 'happy with my purchase reasonably priced thank you '\n",
      " 'these are just great costco no longer carries capers so this brand looked the best and i use them all the time '\n",
      " 'these are great but a little saltier than i like we just wash them off before we use them and it seems to cut out just the right amount of salt for our no salt taste '\n",
      " 'first off i didn t get these via this vendor american delivery service on amazon i picked up a jar of these at my costco a few days ago costco sells this same size jar of this brand for seven and a half bucks so if you like this particular brand of capers it may be worth checking to see if your local costco sells them as to the taste well i ll use them but they are not my favorite brand they have a nice slightly firm flesh but there is something about their brine vinegar that i don t like anywhere near as much as some other brands i use and i probably won t buy them again my current favorites are the bj s house brand berkley jensen which are sold in bj s for about eight bucks per 16 ounce jar those are available via a couple of third party vendors on amazon but again with a hefty markup shipping so it might be worth your time to see if they are in a local bj s '\n",
      " 'we use these in salads almost daily and on chicken and fish they add the pop to many otherwise bland dishes they are delicious and the price is very reasonable '\n",
      " 'this is a great buy and really spices up my cacciatore '\n",
      " 'delicious and reasonable priced this is a repeat order '\n",
      " 'great will order more for all my italian cooking ' 'very good '\n",
      " 'didnt care for it' 'the best'\n",
      " 'i enjoy a few jelly beans as a quick nonfat treat to satisfy my sweet tooth for years i have keep jelly belly black licorice jelly beans on hand but after tasting jelly belly cream soda jelly beans i m hooked when ordering a one pound bag i reviewed my recent purchase history and discovered that i have gobbled five pounds in four months whoa nelly thought it was a big deal when quit drinking sugary cola drinks last year stepped on bathroom scale to check good thing no change i am truly enjoying chewing cream soda beans '\n",
      " 'luv these' 'delicious and sans food dye winner '\n",
      " 'taste great haven t tested in baking yet but look forward to some piece are broken up but what can you expect from sliced almonds '\n",
      " 'great quality at a reasonable price shipped quick slices weren t broken into pieces great taste and you probably can t spend less without a drop in quality '\n",
      " 'these are quite good tasting the price is reasonable as well two pounds should last me a long long time '\n",
      " 'i like these slice almonds they are an essential part of my granola and i will order them again '\n",
      " 'i use almonds in my cereal on my ice cream just by the handful almond are very good for my diet '\n",
      " 'these are great for adding to recipes we eat these all the time we add them to salads asian dishes ice cream baked goods and anything else we can think of that might need a little crunch '\n",
      " 'great price for a huge bag of sliced almonds they arrived fresh and delicious exactly as described very pleased will buy them again when these run out '\n",
      " 'best almonds'\n",
      " 'i use almonds on salads and yogurt these are fantastic and they stay fresh for a long time '\n",
      " 'i have purchased these multiple times if you use a lot of almonds this is a great deal very fresh and tasty i keep the bag in the freezer to maintain freshness since it can take me a while to get through this quantity of almonds '\n",
      " 'large amount and perfect price tast awesome in oat meal cookies and salads etc these tasted very fresh and yes the bag seals but i like to add them into a large mason jar'\n",
      " 'the quality of these almond slices is excellent i use them a lot for baking but they also make good snacks while you re working in the kitchen the price is good too and i plan to buy this brand again '\n",
      " 'mariani has good products for fair prices '\n",
      " 'only place i could find them thanks very fresh and tasty for my salads etc '\n",
      " 'i use these in many types of dishes great '\n",
      " 'quality product but i am now into jordan candy coated almonds i must have something to solve a sweet tooth but i also use mariani sliced almonds as they are good for you '\n",
      " 'these are very good sliced almonds'\n",
      " 'good price great taste what else is there '\n",
      " 'love the sliced almonds on a salad '\n",
      " 'a portion of the product consists of broken pieces but the taste is great and since i use them in my cereal i have eliminated purchasing cereals with nuts can use lower sodium sugar cereal topped with the almonds and some raisins fresh fruit at a lower cost and less additives would recommend and order again '\n",
      " 'this is a wonderful deal '\n",
      " 'these are great i use them in salads i make up snack bags with raisins these walnuts and sometimes dried cranberries i have made some wonderful cookies with these i keep it in the freezer and only keep a small container of them in the fridge to keep them fresh and then just refill the container when i need more originally purchased at costco but my local costco doesn t have them was happy to find them on amazon'\n",
      " 'delicious would recommend and a good quality product'\n",
      " 'very nice crunchy fresh good size bag '\n",
      " 'taste great on salads in granola yogurt and by itself will buy again'\n",
      " 'i ve purchased these before and have been happy every time these are convenient always fresh i keep them in the freezer and remove what i need reseal the bag and buy more when the bag is empty '\n",
      " 'good deal'\n",
      " 'awesome great price great product what more can one say about sliced almonds i would say they re really fresh and good'\n",
      " 'good'\n",
      " 'great taste hard to open as location of zipper vauge pull tabs in each end did not work to open still tough to close so i just dumped in a new zip baggie '\n",
      " 'can t live without them on my greek yogurt ' 'huge nice and fresh'\n",
      " 'put these on everything'\n",
      " 'arrived on time and smelled and tasted fresh i will purchase this product again'\n",
      " 'great' 'great product sliced perfectly good got granola and baking '\n",
      " 'very good and excellent and happy '\n",
      " 'yummy and perfect for baking these are sliced nice and thin and very fresh i have purchased 3 of these and will purchase more my family loves them '\n",
      " 'great almonds for my cereal good price '\n",
      " 'i use a lot of them in baking flavor is great and price is right '\n",
      " 'very good almonds '\n",
      " 'love these almonds i am almost ready to buy another bag i eat them every day for protein in my breakfast oatmeal store them in the freezer to keep them fresh after opening the only bad thing i can say is that the ziplock seal on the bag broke the first time i tried to use it so i just use a chip clip instead '\n",
      " 'love this product but please don t ask for a review every time i buy it '\n",
      " 'nice and fresh'\n",
      " 'i do low carbs these are my go to i do measure the 1 4 cup though but they re fresh and i m sold on them great value for money compared to the almonds in the supermarket '\n",
      " 'delicious with granola '\n",
      " 'they are fresh crunchy and taste great in oatmeal on cool fall days the bag reseals well so they should stay fresh '\n",
      " 'awesome price '\n",
      " 'yummy i love to toast batches and keep them on hand to top cereal yogurt oatmeal anything '\n",
      " 'they are perfect ' 'excellent '\n",
      " 'great very good almonds and perfect to add to salads cereals yogurt '\n",
      " 'what a good price for so many sliced almonds i m a fan '\n",
      " 'i can no longer eat whole almonds due to their being so hard to chew so i thought i d try eating sliced almonds instead since i had previously ordered whole dates by mariani and was very satisfied with them i felt confident that these sliced almonds would also be first quality they are very fresh and crunchy right out of the bag and i can hardly wait to roast them in my oven for an even deeper almondy flavor i m having no adverse dental issues with these fabulous slices of almonds and am looking forward to enjoying my daily ration of 1 4 cup '\n",
      " 'really nice and fresh sliced almonds i use them for breakfast and on salads and have gone through two bags in about 1 2 a year i am on my third bag used to get them in costco but they don t carry it any more so i found it on amazon '\n",
      " 'works as advertised '\n",
      " 'product arrived as promised easy to use in recipes calling for sliced almonds '\n",
      " 'delicious fresh ' 'fresh heathy tasty ' 'very nice '\n",
      " 'handy to add to the morning granola '\n",
      " 'first and foremost i will not purchase again due to learning about the use of ppo only after the purchase my bad and secondly these appear to be a bargain until they arrive and you realized they were most likely bagged to collect the crumbs and pieces left over i could probably find a use for them at that price however the use of ppo when there is clearly a completely safe alternative but not as convenient as spraying with ppo means i won t be buying again and these went in the garbage i won t even feed them to the birds squirrels shame on you and all the rest using ppo and then selling it for consumption and then getting a good nights sleep all the while '\n",
      " 'yummy exactly what i wanted thank you mariani '\n",
      " 'this one of my ingredients in oatmeal cereal '\n",
      " 'great for my diet and very good '\n",
      " 'the product is acceptable and better tasting than other varieties of sliced almond brands however the price is about 50 higher than you can pay at retail '\n",
      " 'great tasting almonds very fresh i like the resealable bag '\n",
      " 'one of the best values around quick delivery quality product '\n",
      " 'these are great i love them on oatmeal they are much better than another brand that i had tried they are crispy and fresh tasting '\n",
      " 'these are great a little less expensive than the thick cut i was ordering from another vendor but these are really good use them in green salads in chicken or tuna salad give a great crunch and nutrition to everyting well worth the cost '\n",
      " 'delicious']\n"
     ]
    }
   ],
   "source": [
    "print(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2count = {} \n",
    "stop = stopwords.words('english')\n",
    "for data in dataset: \n",
    "    words = nltk.word_tokenize(data) \n",
    "    for word in words: \n",
    "        if word not in stop:\n",
    "            if word not in word2count.keys(): \n",
    "                word2count[word] = 1\n",
    "            else: \n",
    "                word2count[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'wait': 3, 'get': 5, 'cooking': 2, 'heard': 1, 'capers': 8, 'never': 1, 'tasted': 3, 'however': 3, 'aster': 1, 'tasting': 7, 'experimenting': 1, 'different': 1, 'recipes': 3, 'large': 4, 'good': 29, 'wonderful': 3, 'quality': 8, 'cheaper': 1, 'use': 26, 'tiny': 1, 'bottles': 1, 'sell': 1, 'stores': 2, 'great': 32, 'tuna': 2, 'sandwiches': 1, 'like': 8, 'potato': 2, 'salad': 4, 'also': 4, 'came': 1, 'nicely': 1, 'packaged': 2, 'prevent': 1, 'breakage': 1, 'buy': 9, 'purchase': 7, 'fantastic': 2, 'product': 12, 'perfect': 5, 'size': 4, 'flavor': 3, 'described': 2, 'thank': 3, 'expert': 1, 'delish': 1, 'always': 2, 'rinse': 1, 'cold': 1, 'water': 2, 'strainer': 1, 'first': 5, 'adding': 2, 'food': 2, 'wash': 2, 'excess': 1, 'salt': 5, 'wish': 1, 'paesana': 1, 'would': 5, 'stop': 1, 'neverending': 1, 'price': 15, 'creep': 1, 'maybe': 1, 'amazon': 6, 'look': 2, 'salty': 5, 'even': 3, 'repeated': 1, 'changing': 1, 'liquid': 1, 'trouble': 1, 'taste': 11, 'anything': 3, 'often': 1, 'couscous': 1, 'lox': 1, 'many': 4, 'dishes': 5, 'simply': 1, 'find': 4, '32': 1, 'oz': 1, 'jars': 1, 'found': 2, 'keep': 9, 'ordering': 3, 'received': 1, 'quickly': 1, 'exactly': 3, 'adertised': 1, 'super': 1, 'sickeningly': 1, 'puckeringly': 1, 'changed': 1, 'brine': 4, 'fresh': 25, 'still': 3, 'new': 2, 'original': 1, 'know': 1, 'almost': 3, 'one': 5, 'gram': 2, '4': 4, 'serving': 1, 'means': 2, '1': 4, 'entire': 1, 'bottle': 1, 'made': 2, 'well': 5, 'tasty': 4, 'several': 1, 'helpings': 1, 'various': 1, 'since': 4, 'arrived': 4, 'glad': 1, 'jar': 5, 'happy': 4, 'reasonably': 1, 'priced': 2, 'costco': 7, 'longer': 2, 'carries': 1, 'brand': 7, 'looked': 1, 'best': 4, 'time': 9, 'little': 3, 'saltier': 1, 'seems': 1, 'cut': 2, 'right': 3, 'amount': 2, 'via': 2, 'vendor': 2, 'american': 1, 'delivery': 2, 'service': 1, 'picked': 1, 'days': 2, 'ago': 1, 'sells': 2, 'seven': 1, 'half': 1, 'bucks': 2, 'particular': 1, 'may': 1, 'worth': 3, 'checking': 1, 'see': 2, 'local': 3, 'favorite': 1, 'nice': 7, 'slightly': 1, 'firm': 1, 'flesh': 1, 'something': 2, 'vinegar': 1, 'anywhere': 1, 'near': 1, 'much': 2, 'brands': 2, 'probably': 3, 'current': 1, 'favorites': 1, 'bj': 3, 'house': 1, 'berkley': 1, 'jensen': 1, 'sold': 2, 'eight': 1, 'per': 1, '16': 1, 'ounce': 1, 'available': 1, 'couple': 1, 'third': 2, 'party': 1, 'vendors': 1, 'hefty': 1, 'markup': 1, 'shipping': 1, 'might': 2, 'salads': 10, 'daily': 2, 'chicken': 2, 'fish': 1, 'add': 5, 'pop': 1, 'otherwise': 1, 'bland': 1, 'delicious': 8, 'reasonable': 4, 'really': 4, 'spices': 1, 'cacciatore': 1, 'repeat': 1, 'order': 4, 'italian': 1, 'didnt': 1, 'care': 1, 'enjoy': 1, 'jelly': 5, 'beans': 4, 'quick': 3, 'nonfat': 1, 'treat': 1, 'satisfy': 1, 'sweet': 2, 'tooth': 2, 'years': 1, 'belly': 2, 'black': 1, 'licorice': 1, 'hand': 2, 'cream': 4, 'soda': 2, 'hooked': 1, 'pound': 1, 'bag': 13, 'reviewed': 1, 'recent': 1, 'history': 1, 'discovered': 1, 'gobbled': 1, 'five': 1, 'pounds': 2, 'four': 1, 'months': 1, 'whoa': 1, 'nelly': 1, 'thought': 2, 'big': 1, 'deal': 4, 'quit': 1, 'drinking': 1, 'sugary': 1, 'cola': 1, 'drinks': 1, 'last': 2, 'year': 2, 'stepped': 1, 'bathroom': 1, 'scale': 1, 'check': 1, 'thing': 2, 'change': 1, 'truly': 1, 'enjoying': 2, 'chewing': 1, 'luv': 1, 'sans': 1, 'dye': 1, 'winner': 1, 'tested': 1, 'baking': 5, 'yet': 1, 'forward': 2, 'piece': 1, 'broken': 3, 'expect': 1, 'sliced': 14, 'almonds': 27, 'shipped': 1, 'slices': 3, 'pieces': 3, 'spend': 1, 'less': 3, 'without': 2, 'drop': 1, 'quite': 1, 'two': 2, 'long': 3, 'slice': 1, 'essential': 1, 'part': 1, 'granola': 5, 'cereal': 6, 'ice': 2, 'handful': 1, 'almond': 3, 'diet': 2, 'eat': 3, 'asian': 1, 'baked': 1, 'goods': 1, 'else': 2, 'think': 1, 'need': 3, 'crunch': 2, 'huge': 2, 'pleased': 1, 'run': 1, 'yogurt': 5, 'stay': 2, 'purchased': 4, 'multiple': 1, 'times': 1, 'lot': 3, 'freezer': 4, 'maintain': 1, 'freshness': 1, 'take': 1, 'quantity': 1, 'tast': 1, 'awesome': 3, 'oat': 1, 'meal': 1, 'cookies': 2, 'etc': 2, 'yes': 1, 'seals': 1, 'mason': 1, 'excellent': 3, 'make': 2, 'snacks': 1, 'working': 1, 'kitchen': 1, 'plan': 1, 'mariani': 4, 'products': 1, 'fair': 1, 'prices': 1, 'place': 1, 'could': 2, 'thanks': 1, 'types': 1, 'jordan': 1, 'candy': 1, 'coated': 1, 'must': 1, 'solve': 1, 'love': 5, 'portion': 1, 'consists': 1, 'eliminated': 1, 'purchasing': 1, 'cereals': 2, 'nuts': 1, 'lower': 2, 'sodium': 1, 'sugar': 1, 'topped': 1, 'raisins': 2, 'fruit': 1, 'cost': 2, 'additives': 1, 'recommend': 2, 'snack': 1, 'bags': 2, 'walnuts': 1, 'sometimes': 1, 'dried': 1, 'cranberries': 1, 'small': 1, 'container': 2, 'fridge': 1, 'refill': 1, 'originally': 1, 'crunchy': 3, 'every': 3, 'convenient': 2, 'remove': 1, 'reseal': 1, 'empty': 1, 'say': 3, 'hard': 2, 'open': 2, 'location': 1, 'zipper': 1, 'vauge': 1, 'pull': 1, 'tabs': 1, 'end': 1, 'work': 1, 'tough': 1, 'close': 1, 'dumped': 1, 'zip': 1, 'baggie': 1, 'live': 1, 'greek': 1, 'put': 1, 'everything': 1, 'smelled': 1, 'perfectly': 1, 'got': 1, 'yummy': 3, 'thin': 1, '3': 1, 'family': 1, 'loves': 1, 'ready': 1, 'another': 3, 'day': 1, 'protein': 1, 'breakfast': 2, 'oatmeal': 5, 'store': 1, 'opening': 1, 'bad': 2, 'ziplock': 1, 'seal': 1, 'broke': 1, 'tried': 2, 'chip': 1, 'clip': 1, 'instead': 2, 'please': 1, 'ask': 1, 'review': 1, 'low': 1, 'carbs': 1, 'go': 1, 'measure': 1, 'cup': 2, 'though': 1, 'value': 1, 'money': 1, 'compared': 1, 'supermarket': 1, 'cool': 1, 'fall': 1, 'reseals': 1, 'toast': 1, 'batches': 1, 'top': 1, 'fan': 1, 'whole': 2, 'due': 2, 'chew': 1, 'try': 1, 'eating': 1, 'previously': 1, 'ordered': 1, 'dates': 1, 'satisfied': 1, 'felt': 1, 'confident': 1, 'hardly': 1, 'roast': 1, 'oven': 1, 'deeper': 1, 'almondy': 1, 'adverse': 1, 'dental': 1, 'issues': 1, 'fabulous': 1, 'looking': 1, 'ration': 1, 'gone': 1, '2': 1, 'used': 1, 'carry': 1, 'works': 1, 'advertised': 1, 'promised': 1, 'easy': 1, 'calling': 1, 'heathy': 1, 'handy': 1, 'morning': 1, 'foremost': 1, 'learning': 1, 'ppo': 4, 'secondly': 1, 'appear': 1, 'bargain': 1, 'arrive': 1, 'realized': 1, 'likely': 1, 'bagged': 1, 'collect': 1, 'crumbs': 1, 'left': 1, 'clearly': 1, 'completely': 1, 'safe': 1, 'alternative': 1, 'spraying': 1, 'buying': 1, 'went': 1, 'garbage': 1, 'feed': 1, 'birds': 1, 'squirrels': 1, 'shame': 1, 'rest': 1, 'using': 1, 'selling': 1, 'consumption': 1, 'getting': 1, 'nights': 1, 'sleep': 1, 'wanted': 1, 'ingredients': 1, 'acceptable': 1, 'better': 2, 'varieties': 1, '50': 1, 'higher': 1, 'pay': 1, 'retail': 1, 'resealable': 1, 'values': 1, 'around': 1, 'crispy': 1, 'expensive': 1, 'thick': 1, 'green': 1, 'give': 1, 'nutrition': 1, 'everyting': 1}\n"
     ]
    }
   ],
   "source": [
    "print(word2count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "freq_words = heapq.nlargest(100, word2count, key=word2count.get)\n",
    "# 100 says that we should only pay attention to the top 100 most frequent words. Change as youd like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = [] \n",
    "for data in dataset: \n",
    "    vector = [] \n",
    "    for word in freq_words: \n",
    "        if word in nltk.word_tokenize(data): \n",
    "            vector.append(1) \n",
    "        else: \n",
    "            vector.append(0) \n",
    "    X.append(vector) \n",
    "X = np.asarray(X) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " ...\n",
      " [1 0 0 ... 0 0 0]\n",
      " [1 1 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "print(X)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}