nep2vec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading XML and TXT corpus simultaneously"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os\n",
    "import unicodedata\n",
    "import nltk\n",
    "import nltk.corpus as nc\n",
    "import NNCCorpus as nnc\n",
    "import string\n",
    "\n",
    "xmlDir = \"C:\\\\Users\\\\uttam\\\\Documents\\\\UMBC\\\\IndependentStudy\\\\dataset\\\\nnc_updated_ah\\\\cs\"\n",
    "xml = nnc.NNCCorpusReader(xmlDir, fileids=r'(?!\\.).*\\.xml')\n",
    "\n",
    "txtDir = \"./data/corpus/\"\n",
    "txt = nc.IndianCorpusReader(txtDir, fileids=r'(?!\\.).*\\.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "    XML corpus clean up\n",
    "'''\n",
    "# Prepare dictionary to remove unnecessary unicode\n",
    "# Not removing HYPHEN and FULL STOP\n",
    "table = dict.fromkeys(i for i in range(sys.maxunicode) \n",
    "                        if unicodedata.category(chr(i)).startswith(('P','N','S','Cf','Cn','Cc'))\n",
    "                        and i != 45 and i != 46)\n",
    "\n",
    "sentences = []\n",
    "counter = 0\n",
    "\n",
    "for sent in xml.sents():\n",
    "    words = []\n",
    "    for word in sent:\n",
    "        result = word.translate(table)\n",
    "        if result != '' and result != '-' and result != '.' and not [i for i in result if all(j in string.whitespace or j in string.punctuation for j in i)]:\n",
    "            words.append(result)\n",
    "    sentences.append(words)\n",
    "# Testing purpose\n",
    "#     counter += 1\n",
    "#     if counter > 10:\n",
    "#         break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "# x = [',,,', '. . .', '123', 'not.number']\n",
    "x = ['. . .']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not [i for i in x if all(j in string.whitespace or j in string.punctuation for j in i)]:\n",
    "    print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['काठमाडौं', 'को', 'केलटोल', 'मा', 'एक', 'मध्यमवर्गीय', 'परिवार', 'को', 'घर', 'थियो'], ['त्यस', 'घर', 'मा', 'श्री', 'भवानीवीर', 'सिंह', 'कंसाकार', 'र', 'श्रीमती', 'लक्ष्मीदेवी', 'कंसाकार', 'का', 'ज्येष्ठ', 'सुपुत्र', 'का', 'रूप', 'मा', 'बैशाख', 'का', 'दिन', 'एक', 'बालक', 'को', 'जन्म', 'भयो', 'जस', 'को', 'नाम', 'राखियो', 'दयावीरसिंह', 'कंसाकार'], ['कंसाकार', 'परिवार', 'बौद्ध', 'धर्मावलम्बी', 'भए', 'को', 'ले', 'बाल्यकाल', 'को', 'वातावरण', 'त्यसै', 'अनुरुप', 'थियो'], ['भगवान्', 'बुद्ध', 'का', 'उपदेश', 'हरु', 'उन', 'का', 'लागि', 'प्रारम्भिक', 'शिक्षा', 'का', 'रुप', 'मा', 'प्राप्त', 'भए'], ['आफ्ना', 'वाल्यवस्था', 'मा', 'उन', 'ले', 'एउ', 'टै', 'शिक्षा', 'प्रवल', 'रुप', 'मा', 'पाए', 'त्यो', 'शिक्षा', 'थियो', 'सेवा', 'नै', 'धर्म', 'हो'], ['घर', 'मा', 'जो', 'जो', 'आउँथे', 'हरेक', 'लाई', 'उन', 'की', 'आमा', 'केही', 'न', 'केही', 'दिएर', 'पठाउँथिन्'], ['कोही', 'पैसा', 'माग्न', 'आउँथे', 'आमा', 'सकेजति', 'दिएर', 'पठाउँथिन्'], ['गरीब', 'को', 'सेवा', 'गर्नु', 'पर्छ', 'उन', 'की', 'आमा', 'भन्ने', 'गर्थिन्'], ['फुर्सद', 'को', 'क्षण', 'मा', 'आमा', 'थैला', 'हरु', 'बनाउँथिन्']]\n"
     ]
    }
   ],
   "source": [
    "# Check the output\n",
    "print(sentences[1:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Word2Vec model and Load it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\uttam\\Anaconda3\\lib\\site-packages\\gensim\\utils.py:1197: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import Word2Vec, KeyedVectors\n",
    "\n",
    "model = Word2Vec(sentences + txt.sents(), size=50, min_count=1, window=5)\n",
    "# model = Word2Vec.load('./large_nep2vec.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = KeyedVectors.load_word2vec_format('./large_nep2vec.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Word2Vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.test.utils import get_tmpfile\n",
    "\n",
    "# Save new model\n",
    "model.wv.save_word2vec_format('large_nep2vec.model', binary=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check for word similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_model = KeyedVectors.load_word2vec_format('large_nep2vec.model', binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('प्रचार-प्रसार', 0.8506048321723938),\n",
       " ('व्याख्या', 0.8461490869522095),\n",
       " ('ग्रहण', 0.841399610042572),\n",
       " ('पक्षपोषण', 0.8406904935836792),\n",
       " ('वकालत', 0.8383564352989197)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model.most_similar('प्रचार', topn=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('छोरा', 0.9441109895706177),\n",
       " ('पत्नी', 0.9349316954612732),\n",
       " ('उनकी', 0.932140052318573),\n",
       " ('बहिनी', 0.9299288392066956),\n",
       " ('पति', 0.9222123622894287),\n",
       " ('भाइ', 0.9216489791870117),\n",
       " ('श्रीमान', 0.9166943430900574),\n",
       " ('बुवा', 0.9166373014450073),\n",
       " ('बाबु', 0.9157494306564331),\n",
       " ('दाजु', 0.913582444190979)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model.most_similar('श्रीमती', topn=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}