PyTorch/rnn_from_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN from scratch\n",
    "\n",
    "---\n",
    "Article: [Building RNNs is Fun with PyTorch and Google Colab](https://medium.com/dair-ai/building-rnns-is-fun-with-pytorch-and-google-colab-3903ea9a3a79)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import os\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will build the following architecture.\n",
    "\n",
    "<img src=\"https://miro.medium.com/max/969/1*o65pRKyHxhw7m8LgMbVERg.png\" alt=\"NN_Architecture\" style=\"width: 500px;\"/>\n",
    "<center>Image from: <a href=\"https://medium.com/dair-ai/building-rnns-is-fun-with-pytorch-and-google-colab-3903ea9a3a79\">Building RNNs is Fun with PyTorch and Google Colab (Aug 19, 2018)</a></center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNN(nn.Module):\n",
    "    def __init__(self, n_inputs, n_neurons):\n",
    "        super(RNN, self).__init__()\n",
    "        \n",
    "        self.wx = torch.randn(n_inputs, n_neurons)\n",
    "        self.wy = torch.randn(n_neurons, n_neurons)\n",
    "        \n",
    "        self.b = torch.zeros(1, n_neurons)\n",
    "        \n",
    "    def forward(self, x0, x1):\n",
    "        self.y0 = torch.tanh(torch.mm(x0, self.wx) + self.b)\n",
    "        self.y1 = torch.tanh(torch.mm(self.y0, self.wy) + torch.mm(x1, self.wx) + self.b)\n",
    "        return self.y0, self.y1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Single neuron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_INPUT = 4\n",
    "N_NEURONS = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x0 = torch.tensor([[0, 1, 2, 0],\n",
    "                  [3, 4, 5, 0],\n",
    "                  [6, 7, 8, 0],\n",
    "                  [9, 0, 1, 0]], dtype=torch.float)\n",
    "x1 = torch.tensor([[9, 8, 7, 0],\n",
    "                  [0, 0, 0, 0],\n",
    "                  [6, 5, 4, 0],\n",
    "                  [3, 2, 1, 0]], dtype=torch.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = RNN(N_INPUT, N_NEURONS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y0, y1 = model(x0, x1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.4530],\n",
      "        [1.0000],\n",
      "        [1.0000],\n",
      "        [1.0000]])\n",
      "tensor([[ 1.0000],\n",
      "        [-0.9544],\n",
      "        [ 1.0000],\n",
      "        [ 1.0000]])\n"
     ]
    }
   ],
   "source": [
    "print('{}\\n{}'.format(y0, y1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Multiple neurons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_INPUT = 3\n",
    "N_NEURONS = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "x0 = torch.tensor([[0, 1, 2],\n",
    "                  [3, 4, 5],\n",
    "                  [6, 7, 8],\n",
    "                  [9, 0, 1]], dtype=torch.float)\n",
    "x1 = torch.tensor([[9, 8, 7],\n",
    "                  [0, 0, 0],\n",
    "                  [6, 5, 4],\n",
    "                  [3, 2, 1]], dtype=torch.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = RNN(N_INPUT, N_NEURONS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "y0, y1 = model(x0, x1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.8887,  0.8353,  0.8815, -0.7926,  0.9857],\n",
      "        [-0.9994,  0.9995,  1.0000, -0.9877,  1.0000],\n",
      "        [-1.0000,  1.0000,  1.0000, -0.9993,  1.0000],\n",
      "        [ 0.6267,  1.0000,  1.0000, -0.9159,  0.9984]])\n",
      "tensor([[-1.0000,  1.0000,  1.0000, -1.0000,  1.0000],\n",
      "        [-0.5925,  0.3841,  0.9867, -0.9946, -0.3306],\n",
      "        [-0.9998,  0.9999,  1.0000, -0.9999,  1.0000],\n",
      "        [-0.9982,  0.9997,  1.0000, -0.2359,  1.0000]])\n"
     ]
    }
   ],
   "source": [
    "print('{}\\n{}'.format(y0 ,y1))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}