js_read.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-09-10T00:08:26.922973",
     "start_time": "2016-09-10T07:08:26.872Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "undefined"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "var fs = require('fs');\n",
    "var split = require('split');\n",
    "var _ = require('lodash');\n",
    "var Hashtags = require('./data/hashtags');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-09-10T00:46:34.056006",
     "start_time": "2016-09-10T07:46:34.050Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Stream {\n",
       "  domain: null,\n",
       "  _events: \n",
       "   { end: [ [Function], [Function] ],\n",
       "     unpipe: [Function: onunpipe],\n",
       "     drain: [Function],\n",
       "     error: [Function: onerror],\n",
       "     close: { [Function: g] listener: [Function: onclose] },\n",
       "     finish: [ [Object], [Function] ],\n",
       "     data: [Function] },\n",
       "  _eventsCount: 7,\n",
       "  _maxListeners: undefined,\n",
       "  writable: true,\n",
       "  readable: true,\n",
       "  paused: false,\n",
       "  autoDestroy: true,\n",
       "  write: [Function],\n",
       "  push: [Function],\n",
       "  queue: [Function],\n",
       "  end: [Function],\n",
       "  destroy: [Function],\n",
       "  pause: [Function],\n",
       "  resume: [Function] }"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ { k: '#demsinphilly', l: 90538 },\n",
      "  { k: '#trump', l: 60891 },\n",
      "  { k: '#imwithher', l: 45621 },\n",
      "  { k: '#tcot', l: 42926 },\n",
      "  { k: '#trump2016', l: 33189 },\n",
      "  { k: '#maga', l: 32915 },\n",
      "  { k: '#dncinphl', l: 28657 },\n",
      "  { k: '#hillaryclinton', l: 26393 },\n",
      "  { k: '#trumptrain', l: 25621 },\n",
      "  { k: '#hillary', l: 24148 },\n",
      "  { k: '#feelthebern', l: 20793 },\n",
      "  { k: '#demconvention', l: 19987 },\n",
      "  { k: '#trumppence16', l: 19826 },\n",
      "  { k: '#makeamericagreatagain', l: 19568 },\n",
      "  { k: '#bernieorbust', l: 17242 } ]\n"
     ]
    }
   ],
   "source": [
    "var logs=[];\n",
    "var hlist;\n",
    "fs.createReadStream('data/2016-07-27/part-00000')\n",
    "    .pipe(split())\n",
    "    .on('data', function (line) {\n",
    "        if(line.length>5){\n",
    "            d=line.replace(/\\]/, '').replace(/\\[/, '').split(',');\n",
    "            user=d.shift()\n",
    "            hashtags=d.shift()\n",
    "            date=d.pop()\n",
    "            location=d.join(',')\n",
    "            hashtags && hashtags.split(\" \").forEach(h=>{\n",
    "                if(h.length>3)\n",
    "                    logs.push({\n",
    "                        hashtag: h,\n",
    "                        date: date\n",
    "                    });\n",
    "            });\n",
    "        }else{\n",
    "            console.log(line)\n",
    "        }\n",
    "    })\n",
    "    .on('end', ()=>{\n",
    "        a=_.groupBy(logs, e=>{return e.hashtag});\n",
    "        hlist=_.sortBy(_.map(a, (v, k)=>{return {k,k, l: v.length}}), l=>{return -l.l}).slice(0,15);\n",
    "        console.log(hlist)\n",
    "    })\n",
    "    .on('finish', ()=>{\n",
    "        console.log(\"finish\")\n",
    "    });"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-09-09T04:28:47.543930",
     "start_time": "2016-09-09T11:28:47.540Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5208333333333334"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "45000/24/60/60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-09-10T00:52:43.908806",
     "start_time": "2016-09-10T07:52:43.902Z"
    },
    "code_folding": [
     0
    ],
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[ { date: '2016-06-06',\n",
       "    hashtags: \n",
       "     [ [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object] ] },\n",
       "  { date: '2016-06-13',\n",
       "    hashtags: \n",
       "     [ [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object] ] },\n",
       "  { date: '2016-07-17',\n",
       "    hashtags: \n",
       "     [ [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object] ] },\n",
       "  { date: '2016-07-18',\n",
       "    hashtags: \n",
       "     [ [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object] ] },\n",
       "  { date: '2016-07-19',\n",
       "    hashtags: \n",
       "     [ [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object] ] },\n",
       "  { date: '2016-07-20',\n",
       "    hashtags: \n",
       "     [ [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object] ] },\n",
       "  { date: '2016-07-24',\n",
       "    hashtags: \n",
       "     [ [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object] ] },\n",
       "  { date: '2016-07-25',\n",
       "    hashtags: \n",
       "     [ [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object] ] },\n",
       "  { date: '2016-07-26',\n",
       "    hashtags: \n",
       "     [ [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object] ] },\n",
       "  { date: '2016-07-27',\n",
       "    hashtags: \n",
       "     [ [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object],\n",
       "       [Object] ] } ]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-09-10T00:50:28.987109",
     "start_time": "2016-09-10T07:50:28.983Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10 ]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_.map(hashtags, d=>{return d.hashtags.length})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Javascript (Node.js)",
   "language": "javascript",
   "name": "javascript"
  },
  "language_info": {
   "file_extension": ".js",
   "mimetype": "application/javascript",
   "name": "javascript",
   "version": "5.0.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}