static_dynamic_lstm_gru.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(0)\n",
    "\n",
    "#创建输入数据  正态分布 2：表示一次的批次数量 4：表示时间序列总数  5：表示具体的数据\n",
    "X = np.random.randn(2,4,5)\n",
    "\n",
    "#第二个样本长度为3\n",
    "X[1,1:] = 0\n",
    "#每一个输入序列的长度\n",
    "seq_lengths = [4,1]\n",
    "print('X:\\n',X)\n",
    "\n",
    "#分别建立一个LSTM与GRU的cell，比较输出的状态  3是隐藏层节点的个数\n",
    "cell = tf.contrib.rnn.BasicLSTMCell(num_units = 3,state_is_tuple = True)\n",
    "gru = tf.contrib.rnn.GRUCell(3)\n",
    "\n",
    "#如果没有initial_state，必须指定a dtype\n",
    "outputs,last_states = tf.nn.dynamic_rnn(cell,X,seq_lengths,dtype =tf.float64 )\n",
    "gruoutputs,grulast_states = tf.nn.dynamic_rnn(gru,X,seq_lengths,dtype =tf.float64 )\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "result,sta,gruout,grusta = sess.run([outputs,last_states,gruoutputs,grulast_states])\n",
    "\n",
    "print('全序列:\\n',result[0])\n",
    "print('短序列:\\n',result[1])\n",
    "\n",
    "#由于在BasicLSTMCell设置了state_is_tuple是True，所以lstm的值为 (状态ct,输出h）\n",
    "print('LSTM的状态:',len(sta),'\\n',sta[1])  \n",
    "\n",
    "print('GRU的全序列：\\n',gruout[0])\n",
    "print('GRU的短序列：\\n',gruout[1])\n",
    "#GRU没有状态输出，其状态就是最终输出，因为批次是两个，所以输出为2\n",
    "print('GRU的状态:',len(grusta),'\\n',grusta[1]) \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_layer_static_lstm(input_x, n_steps, n_hidden):\n",
    "    \n",
    "    '''\n",
    "    返回靜態單層 LSTM 單元的輸出，以及 cell 狀態\n",
    "    args:\n",
    "        input_x: 輸入張量 形狀為 [batch_size, n_steps, n_input]\n",
    "        n_steps: 時序總數\n",
    "        n_hidden：LSTM 單元輸出的節點個數 即隱藏層節點數\n",
    "    '''\n",
    "    \n",
    "    #把輸入 input_x 按列拆分，並返回一個有 n_steps 個張量組成的 list 如 batch_size x 28 x 28 的輸入拆成\n",
    "    #[(batch_size,28),((batch_size,28))....]\n",
    "    #如果是調用的是靜態 rnn 函數，需要這一步處理 即相當於把序列作為第一維度\n",
    "    input_x1 = tf.unstack(input_x, num=n_steps, axis=1)\n",
    "    \n",
    "    #可以看做隱藏層\n",
    "    lstm_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden, forget_bias=1.0)\n",
    "    \n",
    "    #靜態 rnn 函數傳入的是一個張量 list 每一個元素都是一個 (batch_size, n_input)大小的張量\n",
    "    hiddens,states = tf.contrib.rnn.static_rnn(cell=lstm_cell, inputs=input_x1, dtype=tf.float32)\n",
    "\n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_layer_static_lstm(input_x, n_steps, n_hidden):\n",
    "    input_x1 = tf.unstack(input_x, num=n_steps, axis=1)\n",
    "    lstm_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden, forget_bias=1.0)\n",
    "    hiddens,states = tf.contrib.rnn.static_rnn(cell=lstm_cell, inputs=input_x1, dtype=tf.float32)\n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_layer_static_gru(input_x,n_steps,n_hidden):\n",
    "    '''\n",
    "    返回靜態單層 GRU 單元的輸出，以及 cell 狀態\n",
    "    \n",
    "    args:\n",
    "        input_x: 輸入張量 形狀為 [batch_size, n_steps, n_input]\n",
    "        n_steps: 時序總數\n",
    "        n_hidden： gru 單元輸出的節點個數 即隱藏層節點數\n",
    "    '''\n",
    "    #把輸入 input_x 按列拆分，並返回一個有 n_steps 個張量組成的 list 如 batch_size x 28 x 28 的輸入拆成\n",
    "    #[(batch_size,28),((batch_size,28))....]\n",
    "    #如果是調用的是靜態rnn函數，需要這一步處理 即相當於把序列作為第一維度\n",
    "    input_x1 = tf.unstack(input_x,num=n_steps,axis=1)\n",
    "\n",
    "    #可以看做隱藏層\n",
    "    gru_cell = tf.contrib.rnn.GRUCell(num_units=n_hidden)\n",
    "    \n",
    "    #靜態 rnn 函數傳入的是一個張量 list 每一個元素都是一個(batch_size, n_input)大小的張量\n",
    "    hiddens,states = tf.contrib.rnn.static_rnn(cell=gru_cell,inputs=input_x1,dtype=tf.float32)\n",
    "        \n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_layer_static_gru(input_x, n_steps, n_hidden):\n",
    "    input_x1 = tf.unstack(input_x, num=n_steps, axis=1)\n",
    "    gru_cell = tf.contrib.rnn.GRUCell(num_units=n_hidden)\n",
    "    hiddens,states = tf.contrib.rnn.static_rnn(cell=gru_cell, inputs=input_x1, dtype=tf.float32)\n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_layer_dynamic_lstm(input_x,n_steps,n_hidden):\n",
    "    '''\n",
    "    返回動態單層 LSTM 單元的輸出，以及 cell 狀態\n",
    "    \n",
    "    args:\n",
    "        input_x: 輸入張量 形狀為 [batch_size, n_steps, n_input]\n",
    "        n_steps: 時序總數\n",
    "        n_hidden： LSTM 單元輸出的節點個數 即隱藏層節點數\n",
    "    '''\n",
    "    # 可以看做隱藏層\n",
    "    lstm_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias=1.0)\n",
    "    \n",
    "    # 動態 rnn 函數傳入的是一個三維張量，[batch_size, n_steps, n_input]，輸出也是這種形狀\n",
    "    hiddens, states = tf.nn.dynamic_rnn(cell=lstm_cell,inputs=input_x,dtype=tf.float32)\n",
    "\n",
    "    # 注意這裡輸出需要轉置 轉換為時序優先的\n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])\n",
    "    \n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_layer_dynamic_lstm(input_x,n_steps,n_hidden):\n",
    "    lstm_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias=1.0)\n",
    "    hiddens, states = tf.nn.dynamic_rnn(cell=lstm_cell,inputs=input_x,dtype=tf.float32)\n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])\n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_layer_dynamic_gru(input_x,n_steps,n_hidden):\n",
    "    '''\n",
    "    返回動態單層 GRU 單元的輸出，以及 cell 狀態\n",
    "    \n",
    "    args:\n",
    "        input_x: 輸入張量 形狀為 [batch_size, n_steps, n_input]\n",
    "        n_steps: 時序總數\n",
    "        n_hidden： gru 單元輸出的節點個數 即隱藏層節點數\n",
    "    '''\n",
    "    # 可以看做隱藏層\n",
    "    gru_cell = tf.contrib.rnn.GRUCell(num_units=n_hidden)\n",
    "\n",
    "    #動態 rnn 函數傳入的是一個三維張量，[batch_size, n_steps, n_input] 輸出也是這種形狀\n",
    "    hiddens,states = tf.nn.dynamic_rnn(cell=gru_cell,inputs=input_x,dtype=tf.float32)\n",
    "\n",
    "    # 注意這裡輸出需要轉置 轉換為時序優先的\n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])   \n",
    "    \n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_layer_dynamic_gru(input_x,n_steps,n_hidden):\n",
    "    gru_cell = tf.contrib.rnn.GRUCell(num_units=n_hidden)\n",
    "    hiddens,states = tf.nn.dynamic_rnn(cell=gru_cell,inputs=input_x,dtype=tf.float32)\n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])   \n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_static_lstm(input_x,n_steps,n_hidden):\n",
    "    '''\n",
    "    返回靜態多層LSTM單元的輸出，以及cell狀態\n",
    "    \n",
    "    args:\n",
    "        input_x:輸入張量 形狀為[batch_size,n_steps,n_input]\n",
    "        n_steps:時序總數\n",
    "        n_hidden：LSTM單元輸出的節點個數 即隱藏層節點數\n",
    "    '''\n",
    "    \n",
    "    #把輸入 input_x 按列拆分，並返回一個有 n_steps 個張量組成的 list 如 batch_size x 28 x 28 的輸入拆成\n",
    "    #[(batch_size,28),((batch_size,28))....]\n",
    "    #如果是調用的是靜態 rnn 函數，需要這一步處理 即相當於把序列作為第一維度\n",
    "    input_x1 = tf.unstack(input_x,num=n_steps,axis=1)\n",
    "\n",
    "    #可以看做 3 個隱藏層\n",
    "    stacked_rnn = []\n",
    "    for i in range(3):\n",
    "        stacked_rnn.append(tf.contrib.rnn.LSTMCell(num_units=n_hidden))\n",
    "        \n",
    "    #多層RNN的實現 例如cells=[cell1,cell2]，則表示一共有兩層，數據經過cell1後還要經過cells\n",
    "    mcell = tf.contrib.rnn.MultiRNNCell(cells=stacked_rnn)\n",
    "    \n",
    "     #靜態rnn函數傳入的是一個張量list 每一個元素都是一個(batch_size,n_input)大小的張量 \n",
    "    hiddens,states = tf.contrib.rnn.static_rnn(cell=mcell,inputs=input_x1,dtype=tf.float32)\n",
    "\n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_static_lstm(input_x,n_steps,n_hidden):\n",
    "    input_x1 = tf.unstack(input_x,num=n_steps,axis=1)\n",
    "    stacked_rnn = []\n",
    "    for i in range(3):\n",
    "        stacked_rnn.append(tf.contrib.rnn.LSTMCell(num_units=n_hidden))\n",
    "    mcell = tf.contrib.rnn.MultiRNNCell(cells=stacked_rnn)\n",
    "    hiddens,states = tf.contrib.rnn.static_rnn(cell=mcell,inputs=input_x1,dtype=tf.float32)\n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_static_gru(input_x,n_steps,n_hidden):\n",
    "    '''\n",
    "    返回靜態多層GRU單元的輸出，以及cell狀態\n",
    "    args:\n",
    "        input_x:輸入張量 形狀為[batch_size,n_steps,n_input]\n",
    "        n_steps:時序總數\n",
    "        n_hidden：gru單元輸出的節點個數 即隱藏層節點數\n",
    "    '''\n",
    "    \n",
    "    #把輸入 input_x 按列拆分，並返回一個有 n_steps 個張量組成的 list 如 batch_size x 28 x 28 的輸入拆成\n",
    "    #[(batch_size,28),((batch_size,28))....]\n",
    "    #如果是調用的是靜態rnn函數，需要這一步處理 即相當於把序列作為第一維度 \n",
    "    input_x1 = tf.unstack(input_x,num=n_steps,axis=1)\n",
    "\n",
    "    #可以看做 3 個隱藏層\n",
    "    stacked_rnn = []\n",
    "    for i in range(3):\n",
    "        stacked_rnn.append(tf.contrib.rnn.GRUCell(num_units=n_hidden))    \n",
    "        \n",
    "    #多層 RNN 的實現 例如 cells=[cell1,cell2]，則表示一共有兩層，數據經過 cell1 後還要經過 cells\n",
    "    mcell = tf.contrib.rnn.MultiRNNCell(cells=stacked_rnn)\n",
    "    \n",
    "    #靜態rnn函數傳入的是一個張量list 每一個元素都是一個(batch_size,n_input)大小的張量\n",
    "    hiddens,states = tf.contrib.rnn.static_rnn(cell=mcell,inputs=input_x1,dtype=tf.float32)\n",
    "        \n",
    "    return hiddens,states\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_static_gru(input_x,n_steps,n_hidden):\n",
    "    input_x1 = tf.unstack(input_x,num=n_steps,axis=1)\n",
    "    stacked_rnn = []\n",
    "    for i in range(3):\n",
    "        stacked_rnn.append(tf.contrib.rnn.GRUCell(num_units=n_hidden))   \n",
    "    mcell = tf.contrib.rnn.MultiRNNCell(cells=stacked_rnn)\n",
    "    hiddens,states = tf.contrib.rnn.static_rnn(cell=mcell,inputs=input_x1,dtype=tf.float32)\n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_static_mix(input_x,n_steps,n_hidden):\n",
    "    '''\n",
    "    返回靜態多層GRU和LSTM混合單元的輸出，以及cell狀態\n",
    "    \n",
    "    args:\n",
    "        input_x:輸入張量 形狀為[batch_size,n_steps,n_input]\n",
    "        n_steps:時序總數\n",
    "        n_hidden：gru單元輸出的節點個數 即隱藏層節點數\n",
    "    '''\n",
    "\n",
    "    #把輸入 input_x 按列拆分，並返回一個有 n_steps 個張量組成的 list 如 batch_size x 28 x 28 的輸入拆成\n",
    "    #[(batch_size,28),((batch_size,28))....]\n",
    "    #如果是調用的是靜態rnn函數，需要這一步處理 即相當於把序列作為第一維度\n",
    "    input_x1 = tf.unstack(input_x,num=n_steps,axis=1)\n",
    "    \n",
    "    #可以看做2個隱藏層\n",
    "\n",
    "    gru_cell = tf.contrib.rnn.GRUCell(num_units=n_hidden*2)\n",
    "    lstm_cell = tf.contrib.rnn.LSTMCell(num_units=n_hidden)\n",
    "    \n",
    "    #多層RNN的實現 例如cells=[cell1,cell2]，則表示一共有兩層，數據經過cell1後還要經過cells\n",
    "    mcell = tf.contrib.rnn.MultiRNNCell(cells=[lstm_cell,gru_cell])\n",
    "    \n",
    "    #靜態rnn函數傳入的是一個張量list 每一個元素都是一個(batch_size,n_input)大小的張量\n",
    "\n",
    "    hiddens,states = tf.contrib.rnn.static_rnn(cell=mcell,inputs=input_x1,dtype=tf.float32)\n",
    "    \n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_static_mix(input_x,n_steps,n_hidden):\n",
    "    input_x1 = tf.unstack(input_x,num=n_steps,axis=1)\n",
    "    gru_cell = tf.contrib.rnn.GRUCell(num_units=n_hidden*2)\n",
    "    lstm_cell = tf.contrib.rnn.LSTMCell(num_units=n_hidden)\n",
    "    mcell = tf.contrib.rnn.MultiRNNCell(cells=[lstm_cell,gru_cell])\n",
    "    hiddens,states = tf.contrib.rnn.static_rnn(cell=mcell,inputs=input_x1,dtype=tf.float32)\n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_dynamic_lstm(input_x,n_steps,n_hidden):\n",
    "    '''\n",
    "    返回動態多層LSTM單元的輸出，以及cell狀態\n",
    "    \n",
    "    args:\n",
    "        input_x:輸入張量 形狀為[batch_size,n_steps,n_input]\n",
    "        n_steps:時序總數\n",
    "        n_hidden：LSTM單元輸出的節點個數 即隱藏層節點數\n",
    "    '''\n",
    "    #可以看做3個隱藏層\n",
    "    stacked_rnn = []\n",
    "    for i in range(3):\n",
    "        stacked_rnn.append(tf.contrib.rnn.LSTMCell(num_units=n_hidden))\n",
    "        \n",
    "    #多層RNN的實現 例如cells=[cell1,cell2]，則表示一共有兩層，數據經過cell1後還要經過cells\n",
    "    mcell = tf.contrib.rnn.MultiRNNCell(cells=stacked_rnn)\n",
    "    \n",
    "    #動態rnn函數傳入的是一個三維張量，[batch_size,n_steps,n_input] 輸出也是這種形狀\n",
    "    hiddens,states = tf.nn.dynamic_rnn(cell=mcell,inputs=input_x,dtype=tf.float32)\n",
    "    \n",
    "    #注意這裡輸出需要轉置 轉換為時序優先的\n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])    \n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_dynamic_lstm(input_x,n_steps,n_hidden):\n",
    "    stacked_rnn = []\n",
    "    for i in range(3):\n",
    "        stacked_rnn.append(tf.contrib.rnn.LSTMCell(num_units=n_hidden))\n",
    "    mcell = tf.contrib.rnn.MultiRNNCell(cells=stacked_rnn)\n",
    "    hiddens,states = tf.nn.dynamic_rnn(cell=mcell,inputs=input_x,dtype=tf.float32)\n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])    \n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_dynamic_gru(input_x,n_steps,n_hidden):\n",
    "    '''\n",
    "    返回動態多層GRU單元的輸出，以及cell狀態\n",
    "    \n",
    "    args:\n",
    "        input_x:輸入張量 形狀為[batch_size,n_steps,n_input]\n",
    "        n_steps:時序總數\n",
    "        n_hidden：gru單元輸出的節點個數 即隱藏層節點數\n",
    "    '''\n",
    "    #可以看做3個隱藏層\n",
    "    stacked_rnn = []\n",
    "    for i in range(3):\n",
    "        stacked_rnn.append(tf.contrib.rnn.GRUCell(num_units=n_hidden))\n",
    "        \n",
    "    #多層RNN的實現 例如cells=[cell1,cell2]，則表示一共有兩層，數據經過cell1後還要經過cells\n",
    "    mcell = tf.contrib.rnn.MultiRNNCell(cells=stacked_rnn)\n",
    "    \n",
    "    #動態rnn函數傳入的是一個三維張量，[batch_size,n_steps,n_input] 輸出也是這種形狀\n",
    "    hiddens,states = tf.nn.dynamic_rnn(cell=mcell,inputs=input_x,dtype=tf.float32)\n",
    "    \n",
    "    #注意這裡輸出需要轉置 轉換為時序優先的\n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])    \n",
    "    return hiddens,states "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_dynamic_gru(input_x,n_steps,n_hidden):\n",
    "    stacked_rnn = []\n",
    "    for i in range(3):\n",
    "        stacked_rnn.append(tf.contrib.rnn.GRUCell(num_units=n_hidden))\n",
    "    mcell = tf.contrib.rnn.MultiRNNCell(cells=stacked_rnn)\n",
    "    hiddens,states = tf.nn.dynamic_rnn(cell=mcell,inputs=input_x,dtype=tf.float32)\n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])    \n",
    "    return hiddens,states "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_dynamic_mix(input_x,n_steps,n_hidden):\n",
    "    '''\n",
    "    返回動態多層GRU和LSTM混合單元的輸出，以及cell狀態\n",
    "    args:\n",
    "        input_x:輸入張量 形狀為[batch_size,n_steps,n_input]\n",
    "        n_steps:時序總數\n",
    "        n_hidden：gru單元輸出的節點個數 即隱藏層節點數\n",
    "    '''\n",
    "        \n",
    "    #可以看做2個隱藏層\n",
    "    gru_cell = tf.contrib.rnn.GRUCell(num_units=n_hidden*2)\n",
    "    lstm_cell = tf.contrib.rnn.LSTMCell(num_units=n_hidden)\n",
    "    \n",
    "    #多層RNN的實現 例如cells=[cell1,cell2]，則表示一共有兩層，數據經過cell1後還要經過cells\n",
    "    mcell = tf.contrib.rnn.MultiRNNCell(cells=[lstm_cell,gru_cell])\n",
    "    \n",
    "    #動態rnn函數傳入的是一個三維張量，[batch_size,n_steps,n_input] 輸出也是這種形狀\n",
    "    hiddens,states = tf.nn.dynamic_rnn(cell=mcell,inputs=input_x,dtype=tf.float32)\n",
    "    \n",
    "    #注意這裡輸出需要轉置 轉換為時序優先的\n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])    \n",
    "    return hiddens,states\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_dynamic_mix(input_x,n_steps,n_hidden):\n",
    "    gru_cell = tf.contrib.rnn.GRUCell(num_units=n_hidden*2)\n",
    "    lstm_cell = tf.contrib.rnn.LSTMCell(num_units=n_hidden)\n",
    "    mcell = tf.contrib.rnn.MultiRNNCell(cells=[lstm_cell,gru_cell])\n",
    "    hiddens,states = tf.nn.dynamic_rnn(cell=mcell,inputs=input_x,dtype=tf.float32)\n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])    \n",
    "    return hiddens,states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_layer_static_bi_lstm(input_x,n_steps,n_hidden):\n",
    "    '''\n",
    "    返回單層靜態雙向LSTM單元的輸出，以及cell狀態\n",
    "    args:\n",
    "        input_x:輸入張量 形狀為[batch_size,n_steps,n_input]\n",
    "        n_steps:時序總數\n",
    "        n_hidden：LSTM單元輸出的節點個數 即隱藏層節點數\n",
    "    '''\n",
    "    \n",
    "    #把輸入 input_x 按列拆分，並返回一個有 n_steps 個張量組成的 list 如 batch_size x 28 x 28 的輸入拆成\n",
    "    #[(batch_size,28),((batch_size,28))....]\n",
    "    #如果是調用的是靜態rnn函數，需要這一步處理 即相當於把序列作為第一維度 \n",
    "    input_x1 = tf.unstack(input_x,num=n_steps,axis=1)\n",
    "\n",
    "\n",
    "\n",
    "    #正向\n",
    "    lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0)\n",
    "    #反向\n",
    "    lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0)\n",
    "\n",
    "\n",
    "    ##靜態rnn函數傳入的是一個張量list 每一個元素都是一個(batch_size,n_input)大小的張量 這裡的輸出hiddens是一個list 每一個元素都是前向輸出,後向輸出的合併\n",
    "\n",
    "    hiddens,fw_state,bw_state = tf.contrib.rnn.static_bidirectional_rnn(cell_fw=lstm_fw_cell,cell_bw=lstm_bw_cell,inputs=input_x1,dtype=tf.float32)\n",
    "        \n",
    "    print('hiddens:\\n',type(hiddens),len(hiddens),hiddens[0].shape,hiddens[1].shape)    #<class 'list'> 28 (?, 256) (?, 256)\n",
    "    \n",
    "    return hiddens,fw_state,bw_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_layer_static_bi_lstm(input_x,n_steps,n_hidden):\n",
    "    input_x1 = tf.unstack(input_x,num=n_steps,axis=1)\n",
    "    lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0)\n",
    "    lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0)\n",
    "    hiddens,fw_state,bw_state = tf.contrib.rnn.static_bidirectional_rnn(cell_fw=lstm_fw_cell,cell_bw=lstm_bw_cell,\n",
    "                                                                        inputs=input_x1,dtype=tf.float32)\n",
    "    print('hiddens:\\n',type(hiddens),len(hiddens),hiddens[0].shape,hiddens[1].shape)\n",
    "    return hiddens,fw_state,bw_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_layer_dynamic_bi_lstm(input_x,n_steps,n_hidden):\n",
    "    '''\n",
    "    返回單層動態雙向LSTM單元的輸出，以及cell狀態\n",
    "    \n",
    "    args:\n",
    "        input_x:輸入張量 形狀為[batch_size,n_steps,n_input]\n",
    "        n_steps:時序總數\n",
    "        n_hidden：gru單元輸出的節點個數 即隱藏層節點數\n",
    "    '''\n",
    "    \n",
    "    #正向\n",
    "    lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0)\n",
    "    #反向\n",
    "    lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0)\n",
    "\n",
    "    \n",
    "    #動態rnn函數傳入的是一個三維張量，[batch_size,n_steps,n_input] 輸出是一個元組 每一個元素也是這種形狀\n",
    "\n",
    "    hiddens,state = tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,cell_bw=lstm_bw_cell,inputs=input_x,dtype=tf.float32)\n",
    "    \n",
    "    print('hiddens:\\n',type(hiddens),len(hiddens),hiddens[0].shape,hiddens[1].shape)   #<class 'tuple'> 2 (?, 28, 128) (?, 28, 128)\n",
    "    #按axis=2合併 (?,28,128) (?,28,128)按最後一維合併(?,28,256)\n",
    "    hiddens = tf.concat(hiddens,axis=2)\n",
    "    \n",
    "    #注意這裡輸出需要轉置 轉換為時序優先的\n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])    \n",
    "        \n",
    "    return hiddens,state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_layer_dynamic_bi_lstm(input_x,n_steps,n_hidden):\n",
    "    lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0)\n",
    "    lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0)\n",
    "    hiddens,state = tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,cell_bw=lstm_bw_cell,\n",
    "                                                    inputs=input_x,dtype=tf.float32)\n",
    "    print('hiddens:\\n',type(hiddens),len(hiddens),hiddens[0].shape,hiddens[1].shape)\n",
    "    hiddens = tf.concat(hiddens,axis=2)\n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])    \n",
    "    return hiddens,state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_static_bi_lstm(input_x,n_steps,n_hidden):\n",
    "    '''\n",
    "    返回多層靜態雙向LSTM單元的輸出，以及cell狀態\n",
    "    args:\n",
    "        input_x:輸入張量 形狀為[batch_size,n_steps,n_input]\n",
    "        n_steps:時序總數\n",
    "        n_hidden：LSTM單元輸出的節點個數 即隱藏層節點數\n",
    "    '''\n",
    "    \n",
    "    #把輸入 input_x 按列拆分，並返回一個有 n_steps 個張量組成的 list 如 batch_size x 28 x 28 的輸入拆成\n",
    "    #[(batch_size,28),((batch_size,28))....]\n",
    "    #如果是調用的是靜態rnn函數，需要這一步處理 即相當於把序列作為第一維度\n",
    "    input_x1 = tf.unstack(input_x,num=n_steps,axis=1)\n",
    "\n",
    "    stacked_fw_rnn = []\n",
    "    stacked_bw_rnn = []\n",
    "    for i in range(3):\n",
    "        #正向\n",
    "        stacked_fw_rnn.append(tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0))\n",
    "        #反向\n",
    "        stacked_bw_rnn.append(tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0))\n",
    "\n",
    "\n",
    "    \n",
    "    hiddens,fw_state,bw_state = tf.contrib.rnn.stack_bidirectional_rnn(stacked_fw_rnn,stacked_bw_rnn,inputs=input_x1,dtype=tf.float32)\n",
    "        \n",
    "    print('hiddens:\\n',type(hiddens),len(hiddens),hiddens[0].shape,hiddens[1].shape)    #<class 'list'> 28 (?, 256) (?, 256)\n",
    "\n",
    "    return hiddens,fw_state,bw_state\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_static_bi_lstm(input_x,n_steps,n_hidden):\n",
    "    input_x1 = tf.unstack(input_x,num=n_steps,axis=1)\n",
    "    stacked_fw_rnn = []\n",
    "    stacked_bw_rnn = []\n",
    "    for i in range(3):\n",
    "        stacked_fw_rnn.append(tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0))\n",
    "        stacked_bw_rnn.append(tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0))\n",
    "    hiddens,fw_state,bw_state = tf.contrib.rnn.stack_bidirectional_rnn(stacked_fw_rnn,stacked_bw_rnn,\n",
    "                                                                       inputs=input_x1,dtype=tf.float32)  \n",
    "    print('hiddens:\\n',type(hiddens),len(hiddens),hiddens[0].shape,hiddens[1].shape)    \n",
    "    return hiddens,fw_state,bw_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_dynamic_bi_lstm(input_x,n_steps,n_hidden):\n",
    "    '''\n",
    "    返回多層動態雙向LSTM單元的輸出，以及cell狀態\n",
    "    args:\n",
    "        input_x:輸入張量 形狀為[batch_size,n_steps,n_input]\n",
    "        n_steps:時序總數\n",
    "        n_hidden：gru單元輸出的節點個數 即隱藏層節點數\n",
    "    '''    \n",
    "    stacked_fw_rnn = []\n",
    "    stacked_bw_rnn = []\n",
    "    for i in range(3):\n",
    "        #正向\n",
    "        stacked_fw_rnn.append(tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0))\n",
    "        #反向\n",
    "        stacked_bw_rnn.append(tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0))\n",
    "    tf.contrib.rnn.MultiRNNCell\n",
    "    \n",
    "    #動態rnn函數傳入的是一個三維張量，[batch_size,n_steps,n_input] 輸出也是這種形狀，n_input變成了正向和反向合併之後的 即n_input*2\n",
    "    hiddens,fw_state,bw_state = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(stacked_fw_rnn,stacked_bw_rnn,inputs=input_x,dtype=tf.float32)\n",
    "    \n",
    "    print('hiddens:\\n',type(hiddens),hiddens.shape)   # <class 'tensorflow.python.framework.ops.Tensor'> (?, 28, 256)\n",
    "        \n",
    "    #注意這裡輸出需要轉置 轉換為時序優先的\n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])    \n",
    "    \n",
    "    return hiddens,fw_state,bw_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_layer_dynamic_bi_lstm(input_x,n_steps,n_hidden):\n",
    "    stacked_fw_rnn = []\n",
    "    stacked_bw_rnn = []\n",
    "    for i in range(3):\n",
    "        stacked_fw_rnn.append(tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0))\n",
    "        stacked_bw_rnn.append(tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden,forget_bias = 1.0))\n",
    "    tf.contrib.rnn.MultiRNNCell\n",
    "    hiddens,fw_state,bw_state = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(stacked_fw_rnn,stacked_bw_rnn,\n",
    "                                                                               inputs=input_x,dtype=tf.float32)\n",
    "    print('hiddens:\\n',type(hiddens),hiddens.shape)   \n",
    "    hiddens = tf.transpose(hiddens,[1,0,2])    \n",
    "    return hiddens,fw_state,bw_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def  mnist_rnn_classfication(flag):\n",
    "        \n",
    "    '''\n",
    "    1. 導入數據集\n",
    "    '''\n",
    "    tf.reset_default_graph()\n",
    "    from tensorflow.examples.tutorials.mnist import input_data\n",
    "    \n",
    "    # mnist 是一個輕量級的類，它以 numpy 數組的形式存儲著訓練，校驗，測試數據集 one_hot 表示輸出二值化後的 10 維\n",
    "    mnist = input_data.read_data_sets('MNIST-data',one_hot=True)\n",
    "    \n",
    "    print(type(mnist)) #<class 'tensorflow.contrib.learn.python.learn.datasets.base.Datasets'>\n",
    "\n",
    "    print('Training data shape:',mnist.train.images.shape)           #Training data shape: (55000, 784)\n",
    "    print('Test data shape:',mnist.test.images.shape)                #Test data shape: (10000, 784)\n",
    "    print('Validation data shape:',mnist.validation.images.shape)    #Validation data shape: (5000, 784)\n",
    "    print('Training label shape:',mnist.train.labels.shape)          #Training label shape: (55000, 10)\n",
    "    \n",
    "    '''\n",
    "    2 定義參數與網路結構\n",
    "    '''\n",
    "    n_input = 28             # LSTM 單元輸入節點的個數\n",
    "    n_steps = 28             #序列長度\n",
    "    n_hidden = 128           # LSTM 單元輸出節點個數 (即隱藏層個數)\n",
    "    n_classes = 10           #列別\n",
    "    batch_size = 128         #批次大小\n",
    "    training_step = 5000     #迭代次數\n",
    "    display_step  = 200      #幾次顯示一次\n",
    "    learning_rate = 1e-4     #學習率\n",
    "    #定義佔位符\n",
    "    # batch_size：表示一次的批次樣本數量 batch_size \n",
    "    # n_steps：表示時間序列總數 \n",
    "    # n_input： 表示一個時序具體的數據長度 即一共 28 個時序，一個時序送入28個數據進入 LSTM 網絡\n",
    "    input_x = tf.placeholder(dtype=tf.float32,shape=[None,n_steps,n_input])\n",
    "    input_y = tf.placeholder(dtype=tf.float32,shape=[None,n_classes])\n",
    "\n",
    "    if  flag == 1:\n",
    "        print('單層靜態 LSTM 網路：')\n",
    "        hiddens,states = single_layer_static_lstm(input_x,n_steps,n_hidden)\n",
    "    elif flag == 2:\n",
    "        print('單層靜態 gru 網路：')\n",
    "        hiddens,states = single_layer_static_gru(input_x,n_steps,n_hidden)\n",
    "    elif  flag == 3:\n",
    "        print('單層動態 LSTM 網路：')\n",
    "        hiddens,states = single_layer_dynamic_lstm(input_x,n_steps,n_hidden)\n",
    "    elif flag == 4:\n",
    "        print('單層動態 gru 網路：')\n",
    "        hiddens,states = single_layer_dynamic_gru(input_x,n_steps,n_hidden) \n",
    "    elif flag == 5:\n",
    "        print('多層靜態 LSTM 網路：')\n",
    "        hiddens,states = multi_layer_static_lstm(input_x,n_steps,n_hidden)\n",
    "    elif flag == 6:\n",
    "        print('多層靜態 gru 網絡：')\n",
    "        hiddens,states = multi_layer_static_gru(input_x,n_steps,n_hidden)\n",
    "    elif flag == 7:\n",
    "        print('多層靜態 LSTM 和 gru 混合網絡：')\n",
    "        hiddens,states = multi_layer_static_mix(input_x,n_steps,n_hidden)\n",
    "    elif flag == 8:\n",
    "        print('多層動態 LSTM 網絡：')\n",
    "        hiddens,states = multi_layer_dynamic_lstm(input_x,n_steps,n_hidden)\n",
    "    elif flag == 9:\n",
    "        print('多層動態 gru 網絡：')\n",
    "        hiddens,states = multi_layer_dynamic_gru(input_x,n_steps,n_hidden)\n",
    "    elif flag == 10:\n",
    "        print('多層動態 LSTM 和 gru 混合網絡：')\n",
    "        hiddens,states = multi_layer_dynamic_mix(input_x,n_steps,n_hidden)\n",
    "    elif flag == 11:\n",
    "        print('單層靜態雙向 LSTM 網絡：')\n",
    "        hiddens,fw_state,bw_state = single_layer_static_bi_lstm(input_x,n_steps,n_hidden)\n",
    "    elif flag == 12:\n",
    "        print('單層動態雙向 LSTM 網絡：')\n",
    "        hiddens,bw_state = single_layer_dynamic_bi_lstm(input_x,n_steps,n_hidden)\n",
    "    elif flag == 13:\n",
    "        print('多層靜態雙向 LSTM 網絡：')\n",
    "        hiddens,fw_state,bw_state = multi_layer_static_bi_lstm(input_x,n_steps,n_hidden)\n",
    "    elif flag == 14:\n",
    "        print('多層動態雙向 LSTM 網絡：')\n",
    "        hiddens,fw_state,bw_state = multi_layer_dynamic_bi_lstm(input_x,n_steps,n_hidden)\n",
    "    \n",
    "    print('hidden:',hiddens[-1].shape)      #(128,128)\n",
    "    \n",
    "    #取 LSTM 最後一個時序的輸出，然後經過全連接網絡得到輸出值\n",
    "    output = tf.contrib.layers.fully_connected(inputs=hiddens[-1],num_outputs=n_classes,activation_fn = tf.nn.softmax)\n",
    "    '''\n",
    "    3 設置對數似然損失函數\n",
    "    '''\n",
    "    #代價函數 J =-(Σy.logaL)/n    .表示逐元素乘\n",
    "    cost = tf.reduce_mean(-tf.reduce_sum(input_y*tf.log(output),axis=1))\n",
    "    '''\n",
    "    4 求解\n",
    "    '''\n",
    "    train = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
    "    #預測結果評估\n",
    "    #tf.argmax(output,1)  按行統計最大值得索引\n",
    "    correct = tf.equal(tf.argmax(output,1),tf.argmax(input_y,1))       #返回一個數組 表示統計預測正確或者錯誤 \n",
    "    accuracy = tf.reduce_mean(tf.cast(correct,tf.float32))             #求準確率\n",
    "    \n",
    "    #創建 list 保存每一迭代的結果\n",
    "    test_accuracy_list = []\n",
    "    test_cost_list=[]\n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        #使用會話執行圖\n",
    "        sess.run(tf.global_variables_initializer())   #初始化變量    \n",
    "        \n",
    "        #開始迭代 使用Adam優化的隨機梯度下降法\n",
    "        for i in range(training_step): \n",
    "            x_batch,y_batch = mnist.train.next_batch(batch_size = batch_size)   \n",
    "           #Reshape data to get 28 seq of 28 elements\n",
    "            x_batch = x_batch.reshape([-1,n_steps,n_input])\n",
    "            \n",
    "            #開始訓練\n",
    "            train.run(feed_dict={input_x:x_batch,input_y:y_batch})   \n",
    "            if (i+1) % display_step == 0:\n",
    "                 #輸出訓練習準確率\n",
    "                training_accuracy,training_cost = sess.run([accuracy,cost],feed_dict={input_x:x_batch,input_y:y_batch})   \n",
    "                print('Step {0}:Training set accuracy {1},cost {2}.'.format(i+1,training_accuracy,training_cost))\n",
    "        \n",
    "        #全部訓練完成做測試 分成200次，一次測試50個樣本\n",
    "        #輸出測試機準確率 如果一次性全部做測試，內容不夠用會出現OOM錯誤。所以測試時選取比較小的mini_batch來測試\n",
    "        for i in range(200):        \n",
    "            x_batch,y_batch = mnist.test.next_batch(batch_size = 50)      \n",
    "            #Reshape data to get 28 seq of 28 elements\n",
    "            x_batch = x_batch.reshape([-1,n_steps,n_input])\n",
    "            test_accuracy,test_cost = sess.run([accuracy,cost],feed_dict={input_x:x_batch,input_y:y_batch})\n",
    "            test_accuracy_list.append(test_accuracy)\n",
    "            test_cost_list.append(test_cost) \n",
    "            if (i+1)% 20 == 0:\n",
    "                 print('Step {0}:Test set accuracy {1},cost {2}.'.format(i+1,test_accuracy,test_cost)) \n",
    "        print('Test accuracy:',np.mean(test_accuracy_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-34-188c56f277c6>:10: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/user/anaconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/user/anaconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST-data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/user/anaconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST-data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/user/anaconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting MNIST-data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST-data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/user/anaconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "<class 'tensorflow.contrib.learn.python.learn.datasets.base.Datasets'>\n",
      "Training data shape: (55000, 784)\n",
      "Test data shape: (10000, 784)\n",
      "Validation data shape: (5000, 784)\n",
      "Training label shape: (55000, 10)\n",
      "單層靜態 LSTM 網路：\n",
      "hidden: (?, 128)\n",
      "Step 200:Training set accuracy 0.6953125,cost 1.231407642364502.\n",
      "Step 400:Training set accuracy 0.765625,cost 0.7696461081504822.\n",
      "Step 600:Training set accuracy 0.875,cost 0.48919808864593506.\n",
      "Step 800:Training set accuracy 0.875,cost 0.3513914942741394.\n",
      "Step 1000:Training set accuracy 0.8828125,cost 0.42504680156707764.\n",
      "Step 1200:Training set accuracy 0.890625,cost 0.3917798697948456.\n",
      "Step 1400:Training set accuracy 0.953125,cost 0.26326748728752136.\n",
      "Step 1600:Training set accuracy 0.9375,cost 0.2229151725769043.\n",
      "Step 1800:Training set accuracy 0.921875,cost 0.2509880065917969.\n",
      "Step 2000:Training set accuracy 0.953125,cost 0.2702322006225586.\n",
      "Step 2200:Training set accuracy 0.953125,cost 0.1513238102197647.\n",
      "Step 2400:Training set accuracy 0.9453125,cost 0.20558112859725952.\n",
      "Step 2600:Training set accuracy 0.9609375,cost 0.14761097729206085.\n",
      "Step 2800:Training set accuracy 0.9375,cost 0.18940049409866333.\n",
      "Step 3000:Training set accuracy 0.9453125,cost 0.1503661572933197.\n",
      "Step 3200:Training set accuracy 0.921875,cost 0.2628799080848694.\n",
      "Step 3400:Training set accuracy 0.9453125,cost 0.1818096935749054.\n",
      "Step 3600:Training set accuracy 0.9375,cost 0.1601596176624298.\n",
      "Step 3800:Training set accuracy 0.9765625,cost 0.10943764448165894.\n",
      "Step 4000:Training set accuracy 0.9609375,cost 0.1162717193365097.\n",
      "Step 4200:Training set accuracy 0.9765625,cost 0.1000593900680542.\n",
      "Step 4400:Training set accuracy 0.9453125,cost 0.15792179107666016.\n",
      "Step 4600:Training set accuracy 0.9765625,cost 0.15224045515060425.\n",
      "Step 4800:Training set accuracy 0.96875,cost 0.14125007390975952.\n",
      "Step 5000:Training set accuracy 0.984375,cost 0.09865325689315796.\n",
      "Step 20:Test set accuracy 0.9800000190734863,cost 0.06755103170871735.\n",
      "Step 40:Test set accuracy 1.0,cost 0.029333913698792458.\n",
      "Step 60:Test set accuracy 0.8999999761581421,cost 0.23629990220069885.\n",
      "Step 80:Test set accuracy 0.9800000190734863,cost 0.14724430441856384.\n",
      "Step 100:Test set accuracy 0.9800000190734863,cost 0.11856424063444138.\n",
      "Step 120:Test set accuracy 0.9399999976158142,cost 0.11186384409666061.\n",
      "Step 140:Test set accuracy 0.9399999976158142,cost 0.2119673192501068.\n",
      "Step 160:Test set accuracy 0.9599999785423279,cost 0.09280092269182205.\n",
      "Step 180:Test set accuracy 0.9800000190734863,cost 0.1700880080461502.\n",
      "Step 200:Test set accuracy 0.9200000166893005,cost 0.1887889802455902.\n",
      "Test accuracy: 0.96199995\n"
     ]
    }
   ],
   "source": [
    "mnist_rnn_classfication(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}