doc_similarity.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "doc_similarity.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyN82JwQWnFqmb+iGFYGkAr4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/arkaprabha10/ML_models/blob/master/doc_similarity.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l5DXq5uBtvzv"
      },
      "source": [
        "import pandas as pd\r\n",
        "import numpy as np\r\n",
        "from nltk.corpus import stopwords\r\n",
        "from nltk.corpus import wordnet\r\n",
        "from nltk.corpus import stopwords  \r\n",
        "from nltk.tokenize import word_tokenize  \r\n",
        "import nltk\r\n",
        "import re\r\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer \r\n",
        "from sklearn.metrics.pairwise import cosine_similarity\r\n",
        "from sklearn.metrics.pairwise import euclidean_distances\r\n",
        "from sklearn.metrics import jaccard_similarity_score\r\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer \r\n",
        "!pip install -U sentence-transformers\r\n",
        "from sentence_transformers import SentenceTransformer\r\n",
        "from scipy.spatial import distance\r\n",
        "from nltk.corpus import stopwords\r\n",
        "import re\r\n",
        "!pip install language_tool_python\r\n",
        "import language_tool_python\r\n",
        "nltk.download('punkt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bMmf18-qxLgC"
      },
      "source": [
        "documents=['I am a good boy','He is a bad man','Being a good boy doesnt matter in todays world','Cars sucks lamborgini the best']\r\n",
        "documents = [re.sub(r'[^A-Za-z0-9]+', ' ', item) for item in documents]\r\n",
        "documents_df=pd.DataFrame(documents,columns=['documents'])"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ADutZ0SHegq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6d7ff51b-bde7-446e-bf1d-814156f2dbe8"
      },
      "source": [
        "tool = language_tool_python.LanguageTool('en-US')\r\n",
        "# use tokenizer only for cosine similarity\r\n",
        "sbert_model = SentenceTransformer('bert-base-nli-mean-tokens')\r\n"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading LanguageTool: 100%|██████████| 190M/190M [00:10<00:00, 18.8MB/s]\n",
            "Unzipping /tmp/tmpxs8zg2pn.zip to /root/.cache/language_tool_python.\n",
            "Downloaded https://www.languagetool.org/download/LanguageTool-5.2.zip to /root/.cache/language_tool_python.\n",
            "100%|██████████| 405M/405M [00:18<00:00, 21.4MB/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mw4StsI4Cirh"
      },
      "source": [
        "def jaccard_similarity(list1, list2):\r\n",
        "    s1 = set(list1)\r\n",
        "    s2 = set(list2)\r\n",
        "    return float(len(s1.intersection(s2)) / len(s1.union(s2)))"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jZ1TafNPE9F2"
      },
      "source": [
        "def synonym_similarity(list1,list2):\r\n",
        "  for i in list1:\r\n",
        "    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5GorTeN_oxrZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "313b21b9-7514-4ff5-aa6d-069e7ebbe313"
      },
      "source": [
        "\r\n",
        "document_embeddings = sbert_model.encode(documents_df['documents'])\r\n",
        "correct = document_embeddings[0]\r\n",
        "sample1= document_embeddings[1]\r\n",
        "sample1= document_embeddings[2]\r\n",
        "\r\n",
        "# cosine similarity\r\n",
        "c = distance.cosine(correct,sample1)\r\n",
        "\r\n",
        "b = word_tokenize(documents[1])\r\n",
        "a= word_tokenize(documents[0])\r\n",
        "\r\n",
        "# jaccard similarity\r\n",
        "j = jaccard_similarity(a,b)\r\n",
        "\r\n",
        "# grammar check\r\n",
        "matches = tool.check(documents[3])\r\n",
        "grammar_score = max(0,1 - len(matches)/len(documents[0]))\r\n",
        "print(grammar_score)\r\n",
        "print(c,j)\r\n",
        "\r\n",
        "# bigrams for structural similarity\r\n",
        "bigrams=[[b for b in zip(l.split(\" \")[:-1], l.split(\" \")[1:])] for l in documents]\r\n",
        "count = 0\r\n",
        "\r\n",
        "# most_similar(0,pairwise_similarities,'Cosine Similarity')\r\n",
        "# most_similar(0,pairwise_differences,'Euclidean Distance')"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.8666666666666667\n",
            "0.49334603548049927 0.1111111111111111\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7bgBoLmIuQpN"
      },
      "source": [
        "def most_similar(doc_id,similarity_matrix,matrix):\r\n",
        "    print (f'Document: {documents_df.iloc[doc_id][\"documents\"]}')\r\n",
        "    print ('\\n')\r\n",
        "    print (f'Similar Documents using {matrix}:')\r\n",
        "    if matrix=='Cosine Similarity':\r\n",
        "        similar_ix=np.argsort(similarity_matrix[doc_id])[::-1]\r\n",
        "    elif matrix=='Euclidean Distance':\r\n",
        "        similar_ix=np.argsort(similarity_matrix[doc_id])\r\n",
        "    for ix in similar_ix:\r\n",
        "        if ix==doc_id:\r\n",
        "            continue\r\n",
        "        print('\\n')\r\n",
        "        print (f'Document: {documents_df.iloc[ix][\"documents\"]}')\r\n",
        "        print (f'{matrix} : {similarity_matrix[doc_id][ix]}')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}