Seq2Seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_size=256 # # of ASCII Code\n",
    "target_vocab_size=vocab_size # the model selects (classify) one of 256 classes (ASCII codes) per time unit\n",
    "learning_rate=0.1\n",
    "buckets=[(12, 12)] # Because seq2seq does batch learning, it buckets input by length. This time we will only deal with one bucket.\n",
    "PAD=[0] # If the input / target sentence is smaller than the bucket size, pad 0.\n",
    "GO=[1] # Decoder RNN puts the symbol GO as the first input.\n",
    "batch_size=1\n",
    "input_string = \"Hello World\" \n",
    "target_string = \"How are you\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "input_PAD_size = buckets[0][0] - len(input_string) # Decide how much PAD you want to input/\n",
    "target_PAD_size = buckets[0][0] - len(target_string) - 1 # Decide how much PAD you want to target.\n",
    "input_data = (map(ord, input_string) + PAD * input_PAD_size) * batch_size # Change the input text to a list of ASCII codes.\n",
    "target_data = (GO + map(ord, target_string) + PAD * target_PAD_size) * batch_size # Change target phrase to list of ASCII codes.\n",
    "target_weights= ([1.0]*12 + [0.0]*0) * batch_size  # The number of actual valid (loss counted) number of characters \n",
    "                                                    # excluding PAD in the target sentence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class BabySeq2Seq(object):\n",
    "        def __init__(self, source_vocab_size, target_vocab_size, buckets, size, num_layers, batch_size):\n",
    "                self.buckets = buckets\n",
    "                self.batch_size = batch_size\n",
    "                self.source_vocab_size = source_vocab_size\n",
    "                self.target_vocab_size = target_vocab_size\n",
    "\n",
    "                cell = single_cell = tf.contrib.rnn.GRUCell(size)\n",
    "                if num_layers > 1:\n",
    "                 cell = tf.contrib.rnn.MultiRNNCell([single_cell] * num_layers)\n",
    "\n",
    "                # The seq2seq function\n",
    "                # encoder_inputs: A list of ASCII codes in the input sentence.\n",
    "                # decoder_inputs: A list of ASCII codes in the target sentence.\n",
    "                # cell: RNN cell to use for seq2seq.\n",
    "                # num_encoder_symbols, num_decoder_symbols: The number of symbols in the input sentence and the target sentence.\n",
    "                # embedding_size: Size to embed each ASCII code.\n",
    "                # feed_previous: Inference (true for learning / false for Inference)\n",
    "                def seq2seq_f(encoder_inputs, decoder_inputs, do_decode):\n",
    "                        return tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(\n",
    "                                        encoder_inputs, decoder_inputs, cell,\n",
    "                                        num_encoder_symbols=source_vocab_size,\n",
    "                                        num_decoder_symbols=target_vocab_size,\n",
    "                                        embedding_size=size,\n",
    "                                        feed_previous=do_decode)\n",
    "                    \n",
    "                # computational graph \n",
    "                self.encoder_inputs = []\n",
    "                self.decoder_inputs = []\n",
    "                self.target_weights = []\n",
    "                \n",
    "                # Bucket size + one as decoder input node. (One additional creation is because the target symbol is equivalent to the decoder input shifting one space)\n",
    "                for i in xrange(buckets[-1][0]):\n",
    "                    self.encoder_inputs.append(tf.placeholder(tf.int32, shape=[None], name='encoder{0}'.format(i)))\n",
    "\n",
    "                for i in xrange(buckets[-1][1] + 1):\n",
    "                    self.decoder_inputs.append(tf.placeholder(tf.int32, shape=[None], name='decoder{0}'.format(i)))\n",
    "                    self.target_weights.append(tf.placeholder(tf.float32, shape=[None], name='weights{0}'.format(i)))\n",
    "                    \n",
    "                # The target symbol is equivalent to the decoder input shifted by one space.\n",
    "                targets = [self.decoder_inputs[i+1] for i in xrange(len(self.decoder_inputs) - 1)]\n",
    "                \n",
    "                # Using seq2seq with buckets\n",
    "                self.outputs, self.losses = tf.contrib.legacy_seq2seq.model_with_buckets(\n",
    "                                self.encoder_inputs, self.decoder_inputs, targets,\n",
    "                                self.target_weights, buckets,\n",
    "                                lambda x, y: seq2seq_f(x, y, False))\n",
    "                                \n",
    "                # Gradient\n",
    "                self.updates = []\n",
    "                self.updates.append(tf.train.AdamOptimizer(learning_rate).minimize(self.losses[0]))\n",
    "\n",
    "        def step(self, session, encoder_inputs, decoder_inputs, target_weights, test):\n",
    "                bucket_id=0 # Choosing bukcet to use\n",
    "                encoder_size, decoder_size = self.buckets[bucket_id]\n",
    "\n",
    "                # Input feed: encoder inputs, decoder inputs, target_weights \n",
    "                input_feed = {}\n",
    "                for l in xrange(encoder_size):\n",
    "                    input_feed[self.encoder_inputs[l].name] = [encoder_inputs[l]]\n",
    "                for l in xrange(decoder_size):\n",
    "                    input_feed[self.decoder_inputs[l].name] = [decoder_inputs[l]]\n",
    "                    input_feed[self.target_weights[l].name] = [target_weights[l]]\n",
    "\n",
    "                # Insert a value because there is one more decoder input node created.\n",
    "                last_target = self.decoder_inputs[decoder_size].name\n",
    "                input_feed[last_target] = np.zeros([self.batch_size], dtype=np.int32)\n",
    "                last_weight = self.target_weights[decoder_size].name\n",
    "                input_feed[last_weight] = np.zeros([self.batch_size], dtype=np.int32)\n",
    "                \n",
    "\n",
    "                if not test:\n",
    "                        output_feed = [self.updates[bucket_id], self.losses[bucket_id]]\n",
    "                else:\n",
    "                        output_feed = [self.losses[bucket_id]]  # Loss for this batch.\n",
    "                        for l in xrange(decoder_size):  # Output logits.\n",
    "                                output_feed.append(self.outputs[bucket_id][l])\n",
    "\n",
    "\n",
    "                outputs = session.run(output_feed, input_feed)\n",
    "                if not test:\n",
    "                        return outputs[0], outputs[1] # loss\n",
    "                else:\n",
    "                        return outputs[0], outputs[1:] # loss, outputs.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def decode(bytes): # Convert ASCII code to text\n",
    "    return \"\".join(map(chr, bytes)).replace('\\x00', '').replace('\\n', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test():\n",
    "    losses, outputs = model.step(session, input_data, target_data, target_weights, test=True)\n",
    "    words = np.argmax(outputs, axis=2)  # shape (12, 12, 256)\n",
    "    word = decode(words)\n",
    "    print(\"step %d, losses %f, output: Hello World -> %s?\" % (step, losses, word))\n",
    "    if word == \"How are you\": # Stops if the model successfully generate a sentence\n",
    "            print(\">>>>> success! Hello World -> \" + word + \"? <<<<<<<\")\n",
    "            import pdb\n",
    "            pdb.set_trace()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    sess.close()\n",
    "except NameError:\n",
    "    pass\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "step=0\n",
    "test_step=1\n",
    "with tf.Session(config=config) as session:\n",
    "        model= BabySeq2Seq(vocab_size, target_vocab_size, buckets, size=5, num_layers=1, batch_size=batch_size)\n",
    "        session.run(tf.global_variables_initializer())\n",
    "        while True:\n",
    "                model.step(session, input_data, target_data, target_weights, test=False) # no outputs in training\n",
    "                if step % test_step == 0:\n",
    "                        test()\n",
    "                step=step+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}