poem_classification.py

# -*- coding: utf-8 -*-
"""Poem_classification.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1KQSJr8fOS6Onv97j5zV_IbrjS8uPjOAW
"""

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
# %matplotlib inline

#Loading Data
data = pd.read_csv('poem_data.csv')

data.head()

data.info()

data.columns

#Count of Each Genre
data['Genre'].value_counts()

sns.countplot(data['Genre'])

#Spacy for textual Preprocessing

import spacy

nlp = spacy.load('en_core_web_sm')

import string
punc = string.punctuation + '“”|”'

def remove_stop_words(text):
    doc = nlp(text)
    return " ".join([token.lemma_ for token in doc if not token.is_stop])

def to_lower(text):
    return text.lower()

def remove_punc(text):
    for i in punc:
        text = text.replace(i,"")
    return text

# Converting poem to lowercase
data['Poem'] = data['Poem'].apply(to_lower)

# Removing Punctuations
data['Poem'] = data['Poem'].apply(remove_punc)

data.head()

# Tokenization and Lemmatization
data['Tokens'] = data['Poem'].apply(remove_stop_words)

data.head()

# EXtracting features and targets
X = data['Tokens']
y = data['Genre']

#Checking Null values
data.isnull().sum()

from sklearn.model_selection import train_test_split

# Splitting Data into training and testing data
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.25,random_state = 100)

# Importing required Libraries
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn.preprocessing import FunctionTransformer
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV

X_train[0]

# Grid Search CV to find the best parameters
def svc_param_selection(X, y, nfolds):
    Cs = [0.001, 0.01, 0.1, 1, 10]
    gammas = [0.001, 0.01, 0.1, 1]
    param_grid = {'C': Cs, 'gamma' : gammas}
    grid_search = GridSearchCV(SVC(kernel = 'linear'), param_grid, cv=nfolds)
    grid_search.fit(X, y)
    grid_search.best_params_
    return grid_search.best_params_

# using TFIDF on X_train
vec = TfidfVectorizer(min_df=2,max_df=0.95)

X_train = vec.fit_transform(X_train)

# Finding best parameter for SVC()
svc_param_selection(X_train,y_train,20)

# Again Random Split of Data into train and test
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.25,random_state = 100)

# Creating Pipeline -->
poem_clf = Pipeline([
               ("Tfidf", TfidfVectorizer(min_df = 2,max_df = 0.95)),
                ("ToDense",FunctionTransformer(lambda x: x.todense(), accept_sparse=True)),
                ("Classifier",LogisticRegression(penalty = 'l1',C = 1))
])

# Fitting Poem_clf pipeline with data
poem_clf.fit(X_train,y_train)

pred = poem_clf.predict(X_test)

from sklearn.metrics import accuracy_score,confusion_matrix

# Confusion Matrix
print(confusion_matrix(y_test,pred))

print(f"Accuracy for Logistic Regression is: {accuracy_score(y_test,pred) * 100}%")

poem_clf2 = Pipeline([
                ("Tfidf", TfidfVectorizer(min_df = 2,max_df = 0.95)),
                ("ToDense",FunctionTransformer(lambda x: x.todense(), accept_sparse=True)),
                ("Classifier",SVC(kernel = 'linear'))
])

poem_clf2.fit(X_train,y_train)

pred2 = poem_clf2.predict(X_test)

print(confusion_matrix(y_test,pred2))

print(f"Accuracy for SVC is: {round(accuracy_score(y_test,pred2) * 100,3)}%")

"""So we see Logistic Regression is better in this case

Let's try to do topic modelling using Latent Dirichlet Allocation
"""

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation

X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.25,random_state = 100)

X_train.head()

# Vectorization-->
vect = CountVectorizer(min_df = 2,max_df = 0.95,stop_words = 'english')

dtm = vect.fit_transform(X_train)
dtm2 = vect.transform(X_test)

#n_components = 4 for 4 genres
LDA = LatentDirichletAllocation(n_components = 4)

ans = LDA.fit_transform(dtm)
ans2 = LDA.transform(dtm2)

ans.shape

# Prediction of classes according to LDA
Y_train = ans.argmax(axis = 1)
Y_test = ans2.argmax(axis = 1)

poem_LDA_clf = Pipeline([
                ("Tfidf", TfidfVectorizer(min_df = 2,max_df = 0.95)),
                ("ToDense",FunctionTransformer(lambda x: x.todense(), accept_sparse=True)),
                ("Classifier",SVC(gamma = 0.0001,kernel = 'linear'))
])

poem_LDA_clf.fit(X_train,Y_train)

pred3 = poem_LDA_clf.predict(X_test)

print(f'Accuracy using SVC is: {accuracy_score(Y_test,pred3) * 100}%')

poem_LDA_clf2 = Pipeline([
                ("Tfidf", TfidfVectorizer(min_df = 2,max_df = 0.95)),
                ("ToDense",FunctionTransformer(lambda x: x.todense(), accept_sparse=True)),
                ("Classifier",LogisticRegression(penalty = 'l2'))
])

poem_LDA_clf2.fit(X_train,Y_train)

pred4 = poem_LDA_clf2.predict(X_test)

print(f"Accuracy for Logistic Regression is: {accuracy_score(Y_test,pred4) * 100}%")