Analysis on model, submodule and function import.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TODO:\n",
    "# 1. Gather a list of model, submodule and function\n",
    "# 2. Make BQ call to get all python files with sklearn\n",
    "# 3. Use parso to walk through the code, looking for import line\n",
    "# 3.1 not handling import sklearn; a = sklearn.SVM.SVC()?\n",
    "# 3.2 not handling the change of cross_validation (model_selection); cross_validation is treated as a function\n",
    "#     model_selection is a submodule\n",
    "# 4. based on the list, do the count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn_meta_data import metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "models = metadata.getAllModels()\n",
    "submodules = metadata.getAllSubmodules()\n",
    "functions = metadata.getAllFunctions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from bigquery_python_framework.util import run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def excludeByRepo():\n",
    "    from bigquery_python_framework.util import run\n",
    "    allSklearnForks = \"\"\"\\\n",
    "    SELECT\n",
    "        UNIQUE(repo_name)\n",
    "    FROM\n",
    "        [scikit-learn-research.pyfiles.content_py_full]\n",
    "    WHERE\n",
    "        (path CONTAINS \"sklearn\") OR\n",
    "        (repo_name CONTAINS \"scikit-learn\") OR\n",
    "        (repo_name CONTAINS \"sklearn\") OR\n",
    "        (path CONTAINS \"scikit-learn\")\n",
    "    \"\"\"\n",
    "    excludedRepos = [repo_name[0] for repo_name in run(allSklearnForks)]\n",
    "    if not excludedRepos:\n",
    "        return \"\"\n",
    "    s = '(NOT sample_repo_name CONTAINS \"%s\")' % excludedRepos[0]\n",
    "    for repo in excludedRepos[1:]:\n",
    "        s += ' AND '\n",
    "        s += '(NOT sample_repo_name CONTAINS \"%s\")' % repo\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filesContainSklearn = \"\"\"\\\n",
    "SELECT\n",
    "    content\n",
    "FROM\n",
    "    [scikit-learn-research:pyfiles.content_py]\n",
    "WHERE\n",
    "    %s AND\n",
    "    (content CONTAINS \"sklearn\")\n",
    "\"\"\" % excludeByRepo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contents = run(filesContainSklearn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import parso\n",
    "source = \"from sklearn.cross_validation import (ShuffleSplit, cross_val_score,                                       train_test_split)\" #import_from\n",
    "# source = \"import sklearn.SVM.SVC\" #import name\n",
    "node = parso.parse(source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "parso.python.tree.Operator"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(node.children[0].children[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class ImportAnalyzer:\n",
    "    import parso\n",
    "    def __init__(self, library, acceptedList):\n",
    "        from collections import defaultdict\n",
    "        self.library = library\n",
    "        self.acceptedList = acceptedList\n",
    "        self.counter = defaultdict(int)\n",
    "    \n",
    "    def parse(self, code):\n",
    "        node = parso.parse(code)\n",
    "        self.dfs(node)\n",
    "    \n",
    "    def dfs(self, node):\n",
    "        if node.type == 'import_from':\n",
    "            self.handleImportFrom(node)\n",
    "        elif node.type == 'import_name':\n",
    "            self.handleImportName(node)\n",
    "        elif hasattr(node, 'children'):\n",
    "            for child in node.children:\n",
    "                self.dfs(child)\n",
    "        else:\n",
    "            pass\n",
    "    def count(self, value):\n",
    "        if value in self.acceptedList:\n",
    "            self.counter[value] += 1\n",
    "    \n",
    "    def handleImportFrom(self, node):\n",
    "        def getImports(node):\n",
    "            if isinstance(node.children[3], parso.python.tree.Operator) and node.children[3].value == \"(\":\n",
    "                return node.children[4]\n",
    "            else:\n",
    "                return node.children[3]\n",
    "        module = node.children[1]\n",
    "        imports = getImports(node)\n",
    "        if module.type == 'dotted_name' and module.children[0].value == self.library:\n",
    "            # Case 1: from Library.B import C\n",
    "            for dotted_name in module.children:\n",
    "                if dotted_name.type == 'name':\n",
    "                    self.count(dotted_name.value)\n",
    "\n",
    "        if ((module.type == 'name' and module.value == self.library) or\n",
    "            (module.type == 'dotted_name' and module.children[0].value == self.library)):\n",
    "            if imports.type == \"name\":\n",
    "                # Case 2: from Library import A\n",
    "                self.count(imports.value)\n",
    "            elif imports.type == \"import_as_names\":\n",
    "                for child in imports.children:\n",
    "                    if child.type == \"name\":\n",
    "                        # Case 3: from Library import A, B\n",
    "                        self.count(child.value)\n",
    "                    elif child.type == \"import_as_name\":\n",
    "                        # Case 4: from Library import A, B as C\n",
    "                        if child.children[0].type == \"name\":\n",
    "                            self.count(child.children[0].value)\n",
    "            elif imports.type == \"operator\" and imports.value == \"*\":\n",
    "                # Do not handle 'from Library import *' yet\n",
    "                pass\n",
    "            elif imports.type == \"import_as_name\":\n",
    "                if imports.children[0].type == \"name\":\n",
    "                    # Case 5: from Library import A as B\n",
    "                    self.count(imports.children[0].value)\n",
    "            else:\n",
    "                raise Exception(\"Unexpected import line! %r\" % node)\n",
    "    \n",
    "    def handleImportName(self, node):\n",
    "        imports = node.children[1]\n",
    "        if imports.type == 'dotted_name' and imports.children[0] == self.library:\n",
    "            # Case 6: import Library.A\n",
    "            for child in imports.children[1:]:\n",
    "                if child.type == \"name\":\n",
    "                    self.count(child.value)\n",
    "        elif imports.type == 'dotted_as_name' and imports.children[0] == self.library:\n",
    "            # Case 7: import Library.A as B\n",
    "            for child in imports.children[1:]:\n",
    "                if child.type == \"name\":\n",
    "                    self.count(child.value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "modelImportAnalyzer = ImportAnalyzer('sklearn', models + functions + submodules)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "10000\n",
      "10100\n",
      "10200\n",
      "10300\n",
      "10400\n",
      "10500\n",
      "10600\n",
      "10700\n",
      "10800\n",
      "10900\n",
      "11000\n",
      "11100\n",
      "11200\n",
      "11300\n",
      "11400\n",
      "11500\n",
      "11600\n",
      "11700\n",
      "11800\n",
      "11900\n",
      "12000\n",
      "12100\n",
      "12200\n",
      "12300\n",
      "12400\n",
      "12500\n",
      "12600\n",
      "12700\n",
      "12800\n",
      "12900\n",
      "13000\n",
      "13100\n",
      "13200\n",
      "13300\n",
      "13400\n",
      "13500\n",
      "13600\n",
      "13700\n",
      "13800\n",
      "13900\n",
      "14000\n",
      "14100\n",
      "14200\n",
      "14300\n",
      "14400\n",
      "14500\n",
      "14600\n",
      "14700\n",
      "14800\n",
      "14900\n",
      "15000\n",
      "15100\n",
      "15200\n",
      "15300\n",
      "15400\n",
      "15500\n",
      "15600\n",
      "15700\n",
      "15800\n",
      "15900\n",
      "16000\n",
      "16100\n",
      "16200\n",
      "16300\n",
      "16400\n",
      "16500\n",
      "16600\n",
      "16700\n",
      "16800\n",
      "16900\n",
      "17000\n",
      "17100\n",
      "17200\n",
      "17300\n",
      "17400\n",
      "17500\n",
      "17600\n",
      "17700\n",
      "17800\n",
      "17900\n",
      "18000\n",
      "18100\n",
      "18200\n",
      "18300\n",
      "18400\n",
      "18500\n",
      "18600\n",
      "18700\n",
      "18800\n",
      "18900\n",
      "19000\n",
      "19100\n",
      "19200\n",
      "19300\n",
      "19400\n",
      "19500\n",
      "19600\n",
      "19700\n",
      "19800\n",
      "19900\n",
      "20000\n",
      "20100\n",
      "20200\n",
      "20300\n",
      "20400\n",
      "20500\n",
      "20600\n",
      "20700\n",
      "20800\n",
      "20900\n",
      "21000\n",
      "21100\n",
      "21200\n",
      "21300\n",
      "21400\n",
      "21500\n",
      "21600\n",
      "21700\n",
      "21800\n",
      "21900\n",
      "22000\n",
      "22100\n",
      "22200\n",
      "22300\n",
      "22400\n",
      "22500\n",
      "22600\n",
      "22700\n",
      "22800\n",
      "22900\n",
      "23000\n",
      "23100\n",
      "23200\n",
      "23300\n",
      "23400\n",
      "23500\n",
      "23600\n",
      "23700\n",
      "23800\n",
      "23900\n",
      "24000\n",
      "24100\n",
      "24200\n",
      "24300\n",
      "24400\n",
      "24500\n",
      "24600\n",
      "24700\n",
      "24800\n",
      "24900\n",
      "25000\n",
      "25100\n",
      "25200\n",
      "25300\n",
      "25400\n",
      "25500\n",
      "25600\n",
      "25700\n",
      "25800\n",
      "25900\n",
      "26000\n",
      "26100\n",
      "26200\n",
      "26300\n",
      "26400\n",
      "26500\n",
      "26600\n",
      "26700\n",
      "26800\n",
      "26900\n",
      "27000\n",
      "27100\n",
      "27200\n",
      "27300\n",
      "27400\n",
      "27500\n",
      "27600\n",
      "27700\n",
      "27800\n",
      "27900\n",
      "28000\n",
      "28100\n",
      "28200\n",
      "28300\n",
      "28400\n",
      "28500\n",
      "28600\n",
      "28700\n",
      "28800\n",
      "28900\n",
      "29000\n",
      "29100\n",
      "29200\n",
      "29300\n",
      "29400\n",
      "29500\n",
      "29600\n",
      "29700\n",
      "29800\n",
      "29900\n",
      "30000\n",
      "30100\n",
      "30200\n",
      "30300\n",
      "30400\n",
      "30500\n",
      "30600\n",
      "30700\n",
      "30800\n",
      "30900\n",
      "31000\n",
      "31100\n",
      "31200\n",
      "31300\n",
      "31400\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "for i in range(len(contents)):\n",
    "    code = contents[i][0]\n",
    "    modelImportAnalyzer.parse(code)\n",
    "    count += 1\n",
    "    if (count % 100 == 0):\n",
    "        print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def toCSV(dd, filterList, fileName = \"output\"):\n",
    "    # default dict to csv\n",
    "    import operator\n",
    "    L = sorted(modelImportAnalyzer.counter.items(), key=operator.itemgetter(1))\n",
    "    with open(fileName+\".csv\", \"w\") as f:\n",
    "        for model, count in L:\n",
    "            if model in filterList:\n",
    "                f.write(model + \",\" + str(count) + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "toCSV(modelImportAnalyzer.counter, models, \"models\")\n",
    "toCSV(modelImportAnalyzer.counter, submodules, \"submodules\")\n",
    "toCSV(modelImportAnalyzer.counter, functions, \"functions\")"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}