model_decomposable_attention.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.optimizers import Nadam, Adam\n",
    "from keras.layers import *\n",
    "from keras.initializers import *\n",
    "from keras.activations import softmax\n",
    "from keras.layers import InputSpec, Layer, Input, Dense, merge, Conv1D\n",
    "from keras.layers import Lambda, Activation, Dropout, Embedding, TimeDistributed\n",
    "from keras.layers.pooling import GlobalAveragePooling1D, GlobalMaxPooling1D\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "import keras.backend as K\n",
    "from keras.regularizers import l2\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras.models import Model\n",
    "import gensim\n",
    "import gc\n",
    "import os\n",
    "from keras.callbacks import *\n",
    "from keras.optimizers import *\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EpochSaver(gensim.models.callbacks.CallbackAny2Vec):\n",
    "    '''用于保存模型, 打印损失函数等等'''\n",
    "    def __init__(self, savedir, save_name=\"word2vector.model\"):\n",
    "        os.makedirs(savedir, exist_ok=True)\n",
    "        self.save_path = os.path.join(savedir, save_name)\n",
    "        self.epoch = 0\n",
    "        self.pre_loss = 0\n",
    "        self.best_loss = 999999999.9\n",
    "        self.since = time.time()\n",
    "\n",
    "    def on_epoch_end(self, model):\n",
    "        self.epoch += 1\n",
    "        cum_loss = model.get_latest_training_loss() # 返回的是从第一个epoch累计的\n",
    "        epoch_loss = cum_loss - self.pre_loss\n",
    "        time_taken = time.time() - self.since\n",
    "        print(\"Epoch %d, loss: %.2f, time: %dmin %ds\" % \n",
    "                    (self.epoch, epoch_loss, time_taken//60, time_taken%60))\n",
    "        if self.best_loss > epoch_loss:\n",
    "            self.best_loss = epoch_loss\n",
    "            print(\"Better model. Best loss: %.2f\" % self.best_loss)\n",
    "            model.save(self.save_path)\n",
    "            print(\"Model %s save done!\" % self.save_path)\n",
    "\n",
    "        self.pre_loss = cum_loss\n",
    "        self.since = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train =pd.read_csv('/home/kesci/input/bytedance/train_final.csv',names=['qId','q','aId','a','target'],nrows=62500000)\n",
    "test =pd.read_csv('/home/kesci/input/bytedance/bytedance_contest.final_2.csv',names=['qId','q','aId','a'])\n",
    "test['a']=test['a'].apply(lambda x:x[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target=train['target']\n",
    "embed_size=200\n",
    "tokenizer=Tokenizer(lower=False,char_level=False,split=' ')\n",
    "tokenizer.fit_on_texts(train['q'].tolist()+train['a'].tolist()+test['q'].tolist()+test['a'].tolist())\n",
    "train_q=tokenizer.texts_to_sequences(train['q'])\n",
    "train_a=tokenizer.texts_to_sequences(train['a'])\n",
    "test_q=tokenizer.texts_to_sequences(test['q'])\n",
    "test_a=tokenizer.texts_to_sequences(test['a'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train,test\n",
    "gc.collect()\n",
    "maxlen=40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_q=pad_sequences(train_q,maxlen=40,value=0)\n",
    "train_a=pad_sequences(train_a,maxlen=40,value=0)\n",
    "test_q=pad_sequences(test_q,maxlen=40,value=0)\n",
    "test_a=pad_sequences(test_a,maxlen=40,value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_features=750000\n",
    "embedding_matrix=np.zeros((max_features,embed_size))\n",
    "from gensim.models import Word2Vec\n",
    "w2vmodel=Word2Vec.load('/home/kesci/work/w2vfinal_all.model')\n",
    "num_oov=0\n",
    "for word in tokenizer.word_index:\n",
    "    try:\n",
    "        embedding_matrix[tokenizer.word_index[word]]=w2vmodel[word]\n",
    "    except:\n",
    "        num_oov+=1\n",
    "print('oov num is '+str(num_oov))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_fea=pd.read_csv('/home/kesci/work/train_feature_freq_final.csv',nrows=62500000)\n",
    "test_fea=pd.read_csv('/home/kesci/work/test_feature_freq_final.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_fea=train_fea.fillna(0)\n",
    "test_fea=test_fea.fillna(0)\n",
    "train_fea[np.isinf(train_fea)] =0\n",
    "test_fea[np.isinf(test_fea)] =0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unchanged_shape(input_shape):\n",
    "    \"Function for Lambda layer\"\n",
    "    return input_shape\n",
    "\n",
    "\n",
    "def substract(input_1, input_2):\n",
    "    \"Substract element-wise\"\n",
    "    neg_input_2 = Lambda(lambda x: -x, output_shape=unchanged_shape)(input_2)\n",
    "    out_ = Add()([input_1, neg_input_2])\n",
    "    return out_\n",
    "\n",
    "\n",
    "def submult(input_1, input_2):\n",
    "    \"Get multiplication and subtraction then concatenate results\"\n",
    "    mult = Multiply()([input_1, input_2])\n",
    "    sub = substract(input_1, input_2)\n",
    "    out_= Concatenate()([sub, mult])\n",
    "    return out_\n",
    "\n",
    "\n",
    "def apply_multiple(input_, layers):\n",
    "    \"Apply layers to input then concatenate result\"\n",
    "    if not len(layers) > 1:\n",
    "        raise ValueError('Layers list should contain more than 1 layer')\n",
    "    else:\n",
    "        agg_ = []\n",
    "        for layer in layers:\n",
    "            agg_.append(layer(input_))\n",
    "        out_ = Concatenate()(agg_)\n",
    "    return out_\n",
    "\n",
    "\n",
    "def time_distributed(input_, layers):\n",
    "    \"Apply a list of layers in TimeDistributed mode\"\n",
    "    out_ = []\n",
    "    node_ = input_\n",
    "    for layer_ in layers:\n",
    "        node_ = TimeDistributed(layer_)(node_)\n",
    "    out_ = node_\n",
    "    return out_\n",
    "\n",
    "\n",
    "def soft_attention_alignment(input_1, input_2):\n",
    "    \"Align text representation with neural soft attention\"\n",
    "    attention = Dot(axes=-1)([input_1, input_2])\n",
    "    w_att_1 = Lambda(lambda x: softmax(x, axis=1),\n",
    "                     output_shape=unchanged_shape)(attention)\n",
    "    w_att_2 = Permute((2,1))(Lambda(lambda x: softmax(x, axis=2),\n",
    "                             output_shape=unchanged_shape)(attention))\n",
    "    in1_aligned = Dot(axes=1)([w_att_1, input_1])\n",
    "    in2_aligned = Dot(axes=1)([w_att_2, input_2])\n",
    "    return in1_aligned, in2_aligned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decomposable_attention(emb_matrix,maxlen,feature_shape, \n",
    "                           projection_dim=300, projection_hidden=0, projection_dropout=0.2,\n",
    "                           compare_dim=300, compare_dropout=0.2,\n",
    "                           dense_dim=300, dense_dropout=0.2,\n",
    "                           lr=1e-3, activation='elu'):\n",
    "    q1 = Input(name='q1',shape=(maxlen,))\n",
    "    q2 = Input(name='q2',shape=(maxlen,))\n",
    "    \n",
    "\n",
    "    embedding = Embedding(\n",
    "        input_dim=emb_matrix.shape[0],\n",
    "        output_dim=emb_matrix.shape[1],\n",
    "        weights=[emb_matrix],\n",
    "        input_length=maxlen,\n",
    "        trainable=False\n",
    "    )\n",
    "    q1_embed = embedding(q1)\n",
    "    q2_embed = embedding(q2)\n",
    "    \n",
    "    # Projection\n",
    "    projection_layers = []\n",
    "    if projection_hidden > 0:\n",
    "        projection_layers.extend([\n",
    "                Dense(projection_hidden, activation=activation),\n",
    "                Dropout(rate=projection_dropout),\n",
    "            ])\n",
    "    projection_layers.extend([\n",
    "            Dense(projection_dim, activation=None),\n",
    "            Dropout(rate=projection_dropout),\n",
    "        ])\n",
    "    q1_encoded = time_distributed(q1_embed, projection_layers)\n",
    "    q2_encoded = time_distributed(q2_embed, projection_layers)\n",
    "    \n",
    "    # Attention\n",
    "    q1_aligned, q2_aligned = soft_attention_alignment(q1_encoded, q2_encoded)    \n",
    "    \n",
    "    # Compare\n",
    "    q1_combined = Concatenate()([q1_encoded, q2_aligned, submult(q1_encoded, q2_aligned)])\n",
    "    q2_combined = Concatenate()([q2_encoded, q1_aligned, submult(q2_encoded, q1_aligned)]) \n",
    "    compare_layers = [\n",
    "        Dense(compare_dim, activation=activation),\n",
    "        Dropout(compare_dropout),\n",
    "        Dense(compare_dim, activation=activation),\n",
    "        Dropout(compare_dropout),\n",
    "    ]\n",
    "    q1_compare = time_distributed(q1_combined, compare_layers)\n",
    "    q2_compare = time_distributed(q2_combined, compare_layers)\n",
    "    \n",
    "    # Aggregate\n",
    "    q1_rep = apply_multiple(q1_compare, [GlobalAvgPool1D(), GlobalMaxPool1D()])\n",
    "    q2_rep = apply_multiple(q2_compare, [GlobalAvgPool1D(), GlobalMaxPool1D()])\n",
    "    \n",
    "    fea = Input(shape=(feature_shape,))\n",
    "    feax = Dense(128, activation='relu')(fea)\n",
    "    merged = Concatenate()([q1_rep, q2_rep,feax])\n",
    "    dense = Dense(dense_dim, activation=activation)(merged)\n",
    "    dense = Dropout(dense_dropout)(dense)\n",
    "    dense = Dense(dense_dim, activation=activation)(dense)\n",
    "    dense = Dropout(dense_dropout)(dense)\n",
    "    out_ = Dense(1, activation='sigmoid')(dense)\n",
    "    model = Model(inputs=[q1, q2,fea], outputs=out_)\n",
    "    model.compile(optimizer='adam', loss='binary_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_dim=train_fea.shape[1]\n",
    "num_class=1\n",
    "sub1 = np.zeros((test_a.shape[0],1))\n",
    "\n",
    "count = 0\n",
    "folds = KFold(n_splits=5, shuffle=True, random_state=15)\n",
    "for i, (trn_idx, val_idx) in enumerate(folds.split(train_q)):\n",
    "    print(\"FOLD_ \", count + 1)\n",
    "    filepath = \"/home/kesci/work/nnDA_best_model%d.h5\" % count\n",
    "    checkpoint = ModelCheckpoint(\n",
    "        filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n",
    "    reduce_lr = ReduceLROnPlateau(\n",
    "        monitor='val_loss', factor=0.8, patience=1, min_lr=0.00001, verbose=1)\n",
    "    earlystopping = EarlyStopping(\n",
    "        monitor='val_loss', min_delta=0.0001, patience=2, verbose=1, mode='auto')\n",
    "    callbacks = [checkpoint, reduce_lr, earlystopping]\n",
    "    model_nn =decomposable_attention(embedding_matrix, maxlen=maxlen,feature_shape=feature_dim)\n",
    "    X_tr, X_vl, X_tr2, X_vl2, y_tr, y_vl,X_fea_tr,X_fea_vl = train_q[trn_idx], train_q[val_idx], train_a[\n",
    "        trn_idx], train_a[val_idx], target[trn_idx], target[val_idx],train_fea.iloc[trn_idx],train_fea.iloc[val_idx]\n",
    "    hist = model_nn.fit([X_tr, X_tr2,X_fea_tr], y_tr, batch_size=1024, epochs=10, validation_data=([X_vl, X_vl2,X_fea_vl], y_vl),\n",
    "                         callbacks=callbacks, verbose=1, shuffle=True)\n",
    "    print('load_wight')\n",
    "    model_nn.load_weights(filepath)\n",
    "    print('start to predict')\n",
    "    sub1 = model_nn.predict([test_q, test_a,test_fea], batch_size=2024)\n",
    "    break\n",
    "sub=pd.DataFrame(sub1.reshape(-1,1))\n",
    "sub.to_csv('/home/kesci/work/nnda_final.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}