autoencoders.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.colors import LogNorm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup keras/tensorflow\n",
    "import os\n",
    "import tensorflow as tf\n",
    "# config for Gatsby cores\n",
    "if 'gatsby' in os.environ['HOSTNAME']:\n",
    "\n",
    "    # specify target gpu device\n",
    "    os.environ['CUDA_VISIBLE_DEVICES'] = '1' # either '0' or '1' to utilize Titan X GPUs\n",
    "\n",
    "    # allow dynamic GPU allocation\n",
    "    config = tf.ConfigProto()\n",
    "    config.gpu_options.allow_growth = True\n",
    "    session = tf.Session(config=config)\n",
    "    \n",
    "import keras.backend as K\n",
    "from keras import layers\n",
    "from keras.models import Model\n",
    "from keras.utils import to_categorical\n",
    "from keras import objectives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use much faster CuDNNLSTM if possible!\n",
    "use_cudnn = True\n",
    "\n",
    "LSTM = layers.CuDNNLSTM if use_cudnn else layers.LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pick an input file to use for training:\n",
    "\n",
    "#input_file = 'data/npy/mariel-1.npy'\n",
    "input_file = 'data/npy/mariel-2.npy'\n",
    "#input_file = 'data/npy/carrie-10-mins.npy'\n",
    "\n",
    "# Do some simple formatting specific to each dataset.\n",
    "# NB this notebook uses (time, vertex, dimension) axis\n",
    "# ordering throughout.\n",
    "# Uniform scaling is applied so that most sequences of moderate\n",
    "# length can be contained within the range (-1,1) after translation;\n",
    "# this may need to be adjusted for longer sequences.\n",
    "def format_input(input_file):\n",
    "    fname = os.path.basename(input_file)\n",
    "    \n",
    "    X0 = np.load(input_file)\n",
    "\n",
    "    # depending on the input dataset, apply some simple\n",
    "    # transformations and trim appropriate amounts from\n",
    "    # the beginning/end of the file.\n",
    "    if fname == 'mariel-1.npy':\n",
    "        X0 = X0.transpose((1,0,2))\n",
    "        X0 = X0[1300:7000]\n",
    "        X0[:,:,2] *= -1\n",
    "        X0 -= X0.mean(axis=(0,1))\n",
    "        X0 *= 0.5\n",
    "    elif fname == 'mariel-2.npy':\n",
    "        X0 = X0.transpose((1,0,2))\n",
    "        X0 = X0[1500:-1000]\n",
    "        X0[:,:,2] *= -1\n",
    "        X0 -= X0.mean(axis=(0,1))\n",
    "        X0 *= 0.5\n",
    "    elif fname == 'carrie-10-mins.npy':\n",
    "        X0 = X0.transpose((1,0,2))\n",
    "        X0 = X0[100:]\n",
    "        X0[:,:,2] *= -1\n",
    "        X0 -= X0.mean(axis=(0,1))\n",
    "        X0 *= 0.5\n",
    "    else:\n",
    "        print(\"Warning! Don't know how to format file: %s\" % input_file)\n",
    "    \n",
    "    return X0\n",
    "\n",
    "X = format_input(input_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the fraction of data to use for validation, and partition\n",
    "# the input data in time into train/validation sets.\n",
    "\n",
    "validation_split = 0.15\n",
    "\n",
    "# Split the input data into train and validate sets\n",
    "n_train = int((1-validation_split)*X.shape[0])\n",
    "n_val = X.shape[0]-n_train\n",
    "X_train = X[:n_train]\n",
    "X_val = X[n_train:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X[0].mean(axis=0).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define fancy plotting/animation functions\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import mpl_toolkits.mplot3d.axes3d as p3\n",
    "from mpl_toolkits.mplot3d.art3d import juggle_axes\n",
    "from matplotlib import animation\n",
    "from IPython.display import HTML\n",
    "\n",
    "# ask matplotlib to plot up to 2^128 frames in animations\n",
    "matplotlib.rcParams['animation.embed_limit'] = 2**128\n",
    "\n",
    "def update_points(time, points, df):\n",
    "  '''\n",
    "  Callback function called by plotting function below. Mutates the vertex\n",
    "  positions of each value in `points` so the animation moves\n",
    "  @param int time: the index of the time slice to visualize within `df`\n",
    "  @param mpl_toolkits.mplot3d.art3d.Path3DCollection points: the actual\n",
    "    geometry collection whose internal values this function mutates to move\n",
    "    the displayed points\n",
    "  @param numpy.ndarray df: a numpy array with the following three axes:\n",
    "    df.shape[0] = n_vertices\n",
    "    df.shape[1] = n_time_slices\n",
    "    df.shape[2] = n_dimensions\n",
    "  '''\n",
    "  points._offsets3d = juggle_axes(df[time,:,0], df[time,:,1], df[time,:,2], 'z')\n",
    "\n",
    "def animate(seq, frames=None, axis_min=-0.75, axis_max=0.75, speed=45):\n",
    "    if frames is None:\n",
    "        frames = len(seq)\n",
    "    fig = plt.figure()\n",
    "    ax = p3.Axes3D(fig)\n",
    "    ax.set_xlim(axis_min, axis_max)\n",
    "    ax.set_ylim(axis_min, axis_max)\n",
    "    ax.set_zlim(axis_min, axis_max)\n",
    "    points = ax.scatter(seq[0,:,0], seq[0,:,1], seq[0,:,2], depthshade=False)\n",
    "    return animation.FuncAnimation(\n",
    "        fig,\n",
    "        update_points,\n",
    "        frames,\n",
    "        interval=speed,\n",
    "        fargs=(points, seq),\n",
    "        blit=False,\n",
    "    ).to_jshtml()\n",
    "\n",
    "def plot_pose(x, ax=None, lim=(-0.5, 0.5), center=False, colors=None):\n",
    "    if ax is None:\n",
    "        fig = plt.figure()\n",
    "        ax = p3.Axes3D(fig)\n",
    "    if center:\n",
    "        x = x-x.mean(axis=0)\n",
    "    ax.set_xlim(*lim)\n",
    "    ax.set_ylim(*lim)\n",
    "    ax.set_zlim(*lim)\n",
    "    ax.scatter(x[:,0], x[:,1], x[:,2], c=colors)\n",
    "    return ax\n",
    "\n",
    "def plot_stick(x, edges, ax=None, fig=None, subplot=None, lim=(-0.5,0.5), center=False, colors=None):\n",
    "    if subplot is not None:\n",
    "        if fig is None:\n",
    "            fig = plt.figure()\n",
    "        ax = fig.add_subplot(*subplot, projection='3d')\n",
    "    if ax is None:\n",
    "        if fig is None:\n",
    "            fig = plt.figure()\n",
    "        ax = p3.Axes3D(fig)\n",
    "    if center:\n",
    "        x = x-x.mean(axis=0)\n",
    "    ax.set_xlim(*lim)\n",
    "    ax.set_ylim(*lim)\n",
    "    ax.set_zlim(*lim)\n",
    "    #ax.scatter(x[:,0], x[:,1], x[:,2])\n",
    "    for ie,e in enumerate(edges):\n",
    "        if colors is not None:\n",
    "            c = colors[ie]\n",
    "        else:\n",
    "            c = None\n",
    "        ax.plot(np.linspace(x[e[0],0],x[e[1],0],10),np.linspace(x[e[0],1],x[e[1],1],10),np.linspace(x[e[0],2],x[e[1],2],10), color=c)\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate statistics about pairwise distances,\n",
    "# for use in rigidity constraints\n",
    "\n",
    "vdist_mean = np.zeros((X.shape[1],X.shape[1]))\n",
    "vdist_var = np.zeros_like(vdist_mean)\n",
    "vdist_norm = np.zeros_like(vdist_mean)\n",
    "for i in range(X.shape[1]):\n",
    "    for j in range(i+1,X.shape[1]):\n",
    "        vdist = np.sum((X[:,i]-X[:,j])**2, axis=-1)\n",
    "        vdist_mean[i,j] = vdist_mean[j,i] = vdist.mean()\n",
    "        vdist_var[i,j] = vdist_var[j,i] = vdist.var(ddof=1)\n",
    "        vdist_norm[i,j] = vdist_norm[j,i] = np.sqrt(vdist_var[i,j])/vdist_mean[i,j]\n",
    "vdist_std = np.sqrt(vdist_var)\n",
    "\n",
    "vdist_mean_q5, vdist_mean_q95 = np.quantile(vdist_mean[vdist_mean>0], (0.05,0.95))\n",
    "vdist_var_q5, vdist_var_q95 = np.quantile(vdist_var[vdist_var>0], (0.05,0.95))\n",
    "vdist_std_q5, vdist_std_q95 = np.quantile(vdist_std[vdist_std>0], (0.05,0.95))\n",
    "vdist_norm_q5, vdist_norm_q95 = np.quantile(vdist_norm[vdist_norm>0], (0.05, 0.95))\n",
    "print(\"pairwise mean quantiles:  5th=%.2e  95th=%.2e\" % (vdist_mean_q5, vdist_mean_q95))\n",
    "print(\"pairwise std. quantiles:  5th=%.2e  95th=%.2e\" % (vdist_std_q5, vdist_std_q95))\n",
    "plt.figure(figsize=(14,4))\n",
    "plt.subplot(1,3,1)\n",
    "plt.hist(np.log10(vdist_mean[vdist_mean>0]), bins=40);\n",
    "plt.axvline(np.log10(vdist_mean_q5), color='red', lw=1);\n",
    "plt.axvline(np.log10(vdist_mean_q95), color='red', lw=1);\n",
    "plt.xlabel(\"log10(pairwise mean)\", fontsize=14)\n",
    "plt.subplot(1,3,2)\n",
    "plt.hist(np.log10(vdist_std[vdist_std>0]), bins=40);\n",
    "plt.xlabel(\"log10(pairwise std.)\", fontsize=14);\n",
    "plt.axvline(np.log10(vdist_std_q5), color='red', lw=1);\n",
    "plt.axvline(np.log10(vdist_std_q95), color='red', lw=1);\n",
    "plt.subplot(1,3,3)\n",
    "plt.hist((vdist_norm[vdist_norm>0]), bins=40);\n",
    "plt.axvline((vdist_norm_q5), color='red', lw=1);\n",
    "plt.axvline((vdist_norm_q95), color='red', lw=1);\n",
    "plt.xlabel(\"pairwise std/mean\", fontsize=14);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot matrices showing pairwise distance stats\n",
    "plt.figure(figsize=(16,4))\n",
    "plt.subplot(1,3,1)\n",
    "plt.imshow(vdist_mean, vmin=vdist_mean_q5, vmax=vdist_mean_q95, norm=LogNorm())\n",
    "plt.colorbar()\n",
    "plt.title(\"pairwise mean\", fontsize=14)\n",
    "plt.subplot(1,3,2)\n",
    "plt.imshow(vdist_std, vmin=vdist_std_q5, vmax=vdist_mean_q95, norm=LogNorm())\n",
    "plt.title(\"pairwise std.\", fontsize=14);\n",
    "plt.colorbar();\n",
    "plt.subplot(1,3,3)\n",
    "plt.imshow(vdist_norm, vmin=vdist_norm_q5, vmax=vdist_norm_q95, norm=LogNorm())\n",
    "plt.colorbar();\n",
    "plt.title(\"pairwise std/mean\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sort the pairs by increasing pairwise std. dev.\n",
    "upper_triangle = np.triu_indices_from(vdist_var, k=1)\n",
    "variances = vdist_var[upper_triangle]\n",
    "vtx_pairs = sorted(zip(*upper_triangle), key=lambda p: vdist_var[p[0],p[1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# specify the number of rigid connections to use as constraints\n",
    "# for drawing wireframe and/or imposing rigidity constraints\n",
    "# the n_rigid lowest-variance pairs will be considered rigid\n",
    "n_rigid = 150 # 140 seems to work well for mariel-1 and carrie-10; 150 works for mariel-2\n",
    "\n",
    "\n",
    "plt.figure(figsize=(8,3))\n",
    "plt.plot(sorted(vdist_var[upper_triangle]))\n",
    "plt.yscale('log')\n",
    "plt.ylabel('pairwise variance', fontsize=14)\n",
    "plt.xlabel('pair #', fontsize=14)\n",
    "plt.axvline(n_rigid, color='red', lw=1);\n",
    "frame_num = 3500 #500\n",
    "fig = plt.figure(figsize=(8,8))\n",
    "ax = plot_stick(X[frame_num], vtx_pairs[:n_rigid], center=True, fig=fig)\n",
    "plot_pose(X[frame_num], ax, center=True);\n",
    "plt.title(\"Reference pose (frame %d)\"%frame_num, fontsize=16);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Simple autoencoder to learn single-frame poses\n",
    "# Parameters:\n",
    "#   X: input dataset (used to determine shapes)\n",
    "#   n_units: A tuple of 1 or more integers indicating the number of Dense layer units for each dense layer desired.\n",
    "#   add_noise: if True, add unit gaussian noise to each of the encoded layer outputs.\n",
    "#\n",
    "# The returned model requires no target values during training/evaluation\n",
    "# (since the target values are the same as the inputs).\n",
    "def mk_pose_ae(X, n_units=(32,), add_noise=True):\n",
    "    K.clear_session()\n",
    "    \n",
    "    # Define an activation for hidden layers\n",
    "    def s(L):\n",
    "        return layers.PReLU()(L)\n",
    "    \n",
    "    H = encoder_input = layers.Input((X.shape[1], X.shape[2]))\n",
    "    \n",
    "    # Shift the (x,y) values of the input so that it is centered at zero.\n",
    "    # The z-coordinate is left unchanged.\n",
    "    offsets = layers.Lambda(lambda x: K.constant([[[1,1,0]]])*K.mean(x,axis=1,keepdims=True))(encoder_input)\n",
    "    H = layers.Subtract()([H, offsets])\n",
    "    \n",
    "    # Flatten vertices before feeding into dense networks.\n",
    "    H = layers.Reshape((X.shape[1]*X.shape[2],))(H)\n",
    "    \n",
    "    # Create the specified number and size of Dense layers.\n",
    "    for nu in n_units:\n",
    "        H = layers.Dense(nu)(H)\n",
    "        H = s(H)\n",
    "    \n",
    "    # add unit gaussian noise to the latent space, if requested.\n",
    "    if add_noise:\n",
    "        R = K.random_normal(K.shape(H), 0, 1)\n",
    "        H = layers.Lambda(lambda x: x+R)(H)\n",
    "    \n",
    "    # Decoder layers progressively scale the latent space back to\n",
    "    # original input size.\n",
    "    for nu in n_units[::-1][1:]:\n",
    "        H = layers.Dense(nu)(H)\n",
    "        H = s(H)\n",
    "    \n",
    "    # Final dense output layer with tanh activation for (-1,1) range.\n",
    "    H = layers.Dense((X.shape[1]*X.shape[2]), activation='tanh')(H)\n",
    "    H = layers.Reshape((X.shape[1],X.shape[2]))(H)\n",
    "    \n",
    "    # restore the subtracted (x,y) offset before outputting\n",
    "    H = layers.Add()([H, offsets])\n",
    "    decoder_output = H\n",
    "    \n",
    "    autoencoder = Model(encoder_input, decoder_output)\n",
    "    \n",
    "    # Define the autoencoder loss as the pointwise mean squared error of the\n",
    "    # output relative to the input.\n",
    "    ae_loss = K.mean(K.sum(K.square(decoder_output-encoder_input), axis=-1))\n",
    "    autoencoder.add_loss(ae_loss)\n",
    "    \n",
    "    # compile model without target y values (loss is defined by input and output layers only).\n",
    "    autoencoder.compile(optimizer='adam')\n",
    "    \n",
    "    return autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate network with whatever parameters you like!\n",
    "\n",
    "n_units = (64,)\n",
    "add_noise = True\n",
    "\n",
    "auto = mk_pose_ae(X, n_units=n_units, add_noise=add_noise)\n",
    "auto.summary()\n",
    "\n",
    "# these will get populated after each call to fit() by the values in the\n",
    "# built-in History callback.\n",
    "losses = []\n",
    "val_losses = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# specify training parameters and fit the model.\n",
    "# Note: subjectively, it seems that a loss of 2e-3 looks fairly convincing.\n",
    "batch_size = 128\n",
    "epochs = 16\n",
    "learning_rate = 3e-4\n",
    "\n",
    "\n",
    "K.set_value(auto.optimizer.lr, learning_rate)\n",
    "\n",
    "auto.fit(X_train, None, validation_data=(X_val, None),\n",
    "         epochs=epochs, batch_size=batch_size)\n",
    "\n",
    "losses.extend(auto.history.history['loss'])\n",
    "val_losses.extend(auto.history.history['val_loss'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot losses and some example auto-encoded poses from the validation set\n",
    "plt.figure(figsize=(8,4))\n",
    "plt.plot(losses, label='train')\n",
    "plt.plot(val_losses, label='val')\n",
    "plt.yscale('log')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('loss');\n",
    "\n",
    "n_sample = 2\n",
    "itest = np.random.randint(0,len(X_val),n_sample)\n",
    "xtest = X_val[itest]\n",
    "ytest = auto.predict(xtest)\n",
    "for i in range(n_sample):\n",
    "    fig = plt.figure(figsize=(10,6))\n",
    "    plt.suptitle(\"Validation frame %d\"% itest[i])\n",
    "    plot_stick(xtest[i]-xtest[i].mean(axis=0), vtx_pairs[:n_rigid], center=True, fig=fig, subplot=(1,2,1))\n",
    "    plot_stick(ytest[i]-ytest[i].mean(axis=0), vtx_pairs[:n_rigid], center=True, fig=fig, subplot=(1,2,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now let's try to autoencode a whole sequence of poses.\n",
    "def mk_seq_ae(X, seq_len=32, encoded_dim=32, lstm_units=(128,), add_noise=True):\n",
    "    K.clear_session()\n",
    "    \n",
    "    noise_layer = layers.Lambda(lambda x: x+K.random_normal(K.shape(x)))\n",
    "    \n",
    "    \n",
    "    autoencoder_input = H = layers.Input((seq_len, X.shape[1],X.shape[2],))\n",
    "    \n",
    "    # as before, we shift the entire sequence in (x,y) so that it's centered about (0,0)\n",
    "    offsets = layers.Lambda(lambda x: K.constant([[[[1,1,0]]]])*K.mean(x,axis=(1,2),keepdims=True))(autoencoder_input)\n",
    "    H = layers.Subtract()([H, offsets])\n",
    "    H = layers.Reshape((seq_len, X.shape[1]*X.shape[2]))(H)\n",
    "    \n",
    "    # now encode the poses\n",
    "    H = layers.Dense(encoded_dim)(H)\n",
    "    H = layers.PReLU()(H)\n",
    "    if add_noise:\n",
    "        H = noise_layer(H)\n",
    "    \n",
    "    # branch off a secondary subnetwork that just immediately decodes\n",
    "    # the sequence of decoded poses\n",
    "    H2 = H\n",
    "    H2 = layers.Dense(X.shape[1]*X.shape[2], activation='tanh')(H2)\n",
    "    H2 = layers.Reshape((seq_len, X.shape[1],X.shape[2]))(H2)\n",
    "    H2 = layers.Add()([H2, offsets])\n",
    "    autoencoded_poses = H2\n",
    "        \n",
    "    # feed into LSTM(s)\n",
    "    for nu in lstm_units:\n",
    "        H = LSTM(nu, return_sequences=True)(H)\n",
    "    \n",
    "    # decode the poses in the LSTM sequence\n",
    "    H = layers.Dense(X.shape[1]*X.shape[2], activation='tanh')(H)\n",
    "    \n",
    "    # un-shift the offsets subtracted in the beginning:\n",
    "    H = layers.Reshape((seq_len, X.shape[1], X.shape[2]))(H)\n",
    "    H = layers.Add()([H, offsets])\n",
    "    autoencoder_output = H\n",
    "    \n",
    "    auto = Model(autoencoder_input, [autoencoder_output, autoencoded_poses])\n",
    "    \n",
    "    auto.hp_seq_weight = K.variable(1.0)\n",
    "    auto.hp_pose_weight = K.variable(1.0)\n",
    "    \n",
    "    # define autoencoding loss as the vertex-wise mean squared error of the output vs. input\n",
    "    ae_loss = K.mean(K.sum(K.square(autoencoder_input-autoencoder_output), axis=-1))\n",
    "    auto.add_loss(auto.hp_seq_weight*ae_loss)\n",
    "    \n",
    "    pose_ae_loss = K.mean(K.sum(K.square(autoencoder_input-autoencoded_poses), axis=-1))\n",
    "    auto.add_loss(auto.hp_pose_weight*pose_ae_loss)\n",
    "    \n",
    "    auto.compile(optimizer='adam')\n",
    "    \n",
    "    return auto"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generator function to sample batches of contiguous sequences from a given dataset\n",
    "def gen_batches(X, batch_size, seq_len):\n",
    "    batch_idxs = np.arange(X.shape[0]-seq_len).repeat(seq_len).reshape(-1,seq_len) + np.arange(seq_len)\n",
    "    nbatch = batch_idxs.shape[0]//batch_size\n",
    "    while True:\n",
    "        np.random.shuffle(batch_idxs)\n",
    "        for ibatch in range(nbatch):\n",
    "            #yield X[batch_idxs[ibatch*nbatch:(ibatch+1)*nbatch]], None\n",
    "            yield X[batch_idxs[ibatch*batch_size:(ibatch+1)*batch_size]], None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# specify training parameters and fit the model.\n",
    "seq_len = 64\n",
    "encoded_dim = 32\n",
    "lstm_units = (16,)\n",
    "add_noise = True\n",
    "\n",
    "auto = mk_seq_ae(X, seq_len=seq_len, encoded_dim=encoded_dim, lstm_units=lstm_units, add_noise=add_noise)\n",
    "\n",
    "losses = []\n",
    "val_losses = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "epochs = 8\n",
    "learning_rate = 1e-3\n",
    "seq_weight = 1.0\n",
    "pose_weight = 0.0\n",
    "\n",
    "K.set_value(auto.optimizer.lr, learning_rate)\n",
    "K.set_value(auto.hp_seq_weight, seq_weight)\n",
    "K.set_value(auto.hp_pose_weight, pose_weight)\n",
    "\n",
    "nbatch_train = X_train.shape[0]//batch_size\n",
    "nbatch_val = X_val.shape[0]//batch_size\n",
    "\n",
    "auto.fit_generator(gen_batches(X_train, batch_size, seq_len),\n",
    "                   validation_data=gen_batches(X_val, batch_size, seq_len),\n",
    "                   validation_steps=nbatch_val,\n",
    "                   steps_per_epoch=nbatch_train, epochs=epochs)\n",
    "\n",
    "losses.extend(auto.history.history['loss'])\n",
    "val_losses.extend(auto.history.history['val_loss'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(losses[2:])\n",
    "plt.plot(val_losses[2:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pick a sequence from the validation set to visualize autoencoding performance\n",
    "reference_sequence = X_val[640:640+seq_len]\n",
    "\n",
    "seq_pred, pose_pred = auto.predict(np.expand_dims(reference_sequence,axis=0))\n",
    "HTML(animate(seq_pred[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "HTML(animate(pose_pred[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "HTML(animate(reference_sequence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}