Train-LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
    "import tensorflow as tf\n",
    "import glob\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)\n",
    "import numpy as np\n",
    "#from tensorboardX import SummaryWriter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from youtube-8m utils.py\n",
    "def Dequantize(feat_vector, max_quantized_value=2, min_quantized_value=-2):\n",
    "  \"\"\"Dequantize the feature from the byte format to the float format.\n",
    "\n",
    "  Args:\n",
    "    feat_vector: the input 1-d vector.\n",
    "    max_quantized_value: the maximum of the quantized value.\n",
    "    min_quantized_value: the minimum of the quantized value.\n",
    "\n",
    "  Returns:\n",
    "    A float vector which has the same shape as feat_vector.\n",
    "  \"\"\"\n",
    "  assert max_quantized_value > min_quantized_value\n",
    "  quantized_range = max_quantized_value - min_quantized_value\n",
    "  scalar = quantized_range / 255.0\n",
    "  bias = (quantized_range / 512.0) + min_quantized_value\n",
    "  return feat_vector * scalar + bias\n",
    "\n",
    "def decode(feat_vector, feature_size):\n",
    "    return tf.reshape(tf.cast(tf.decode_raw(feat_vector, \n",
    "                                            tf.uint8), \n",
    "                              tf.float32),\n",
    "                      [-1, feature_size])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filepath is path to tfrecord\n",
    "# datatype is audio or video\n",
    "# output_features and output_labels are empty lists or existing lists\n",
    "def load_data(filepath, data_type, output_labels, output_features):\n",
    "    if data_type == 'audio':\n",
    "        context = {\n",
    "            'labels': tf.VarLenFeature(dtype=tf.int64)\n",
    "        }\n",
    "\n",
    "        feature_list = {\n",
    "            'audio_embedding': tf.FixedLenSequenceFeature([], dtype=tf.string)\n",
    "        }\n",
    "        feature_name = 'audio_embedding'\n",
    "        feature_len = 128\n",
    "\n",
    "    elif data_type == 'video':\n",
    "        context = {\n",
    "            'id': tf.FixedLenFeature([], dtype=tf.string),\n",
    "            'labels': tf.VarLenFeature(dtype=tf.int64)\n",
    "        }\n",
    "\n",
    "        feature_list = {\n",
    "            'rgb': tf.FixedLenSequenceFeature([], dtype=tf.string),\n",
    "        }\n",
    "        feature_name = 'rgb'\n",
    "        feature_len = 128\n",
    "        \n",
    "        \n",
    "    tf.reset_default_graph()    \n",
    "\n",
    "\n",
    "    # Read TFRecord file\n",
    "    reader = tf.TFRecordReader()\n",
    "    filename_queue = tf.train.string_input_producer([filepath])\n",
    "\n",
    "\n",
    "    # Extract features from serialized data\n",
    "\n",
    "    _, serialized_example = reader.read(filename_queue)\n",
    "    context, features = tf.io.parse_single_sequence_example(serialized_example,\n",
    "                                                    context_features=context,\n",
    "                                                    sequence_features=feature_list,\n",
    "                                                    example_name=None,\n",
    "                                                    name=None\n",
    "    )\n",
    "    labels = context['labels']\n",
    "    label = labels.values[0]\n",
    "    data = Dequantize(decode(features[feature_name], feature_len))\n",
    "\n",
    "    # Many tf.train functions use tf.train.QueueRunner,\n",
    "    # so we need to start it before we read\n",
    "    \n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.local_variables_initializer())\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        coord = tf.train.Coordinator()\n",
    "        threads = tf.train.start_queue_runners(sess=sess, coord=coord)\n",
    "        # f = codecs.open(outpath, \"w\", encoding='utf-8')\n",
    "        try:\n",
    "            counter = 0\n",
    "            recordlist = []\n",
    "\n",
    "            num_in_file = sum(1 for _ in tf.python_io.tf_record_iterator(filepath))\n",
    "\n",
    "            for i in range(num_in_file):\n",
    "                d, l = sess.run([data, label])\n",
    "                output_labels.append(l)\n",
    "                output_features.append(d)\n",
    "        except tf.errors.OutOfRangeError:\n",
    "            print('Finished extracting from tfrecord data.')\n",
    "        finally:\n",
    "            coord.request_stop()\n",
    "            coord.join(threads)\n",
    "\n",
    "\n",
    "    \n",
    "    return output_labels, output_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_output_labels = []\n",
    "audio_output_features = []\n",
    "audio_path = \"audio_1556878148.412379.tfrecord\"\n",
    "audio_output_labels, audio_output_features = load_data(audio_path,\n",
    "                                           'audio', audio_output_labels, audio_output_features)\n",
    "\n",
    "video_output_labels = []\n",
    "video_output_features = []\n",
    "video_path = \"video_1556875795.817071.tfrecord\"\n",
    "video_output_labels, video_output_features = load_data(video_path,\n",
    "                                           'video', video_output_labels, video_output_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "audio_x shape:  (2243, 10, 128)\n",
      "audio_y shape:  (2243,)\n",
      "video_x shape:  (2243, 10, 128)\n",
      "video_y shape:  (2243,)\n"
     ]
    }
   ],
   "source": [
    "audio_x = np.array(audio_output_features)\n",
    "audio_y = np.array(audio_output_labels)\n",
    "video_x = np.array(video_output_features)\n",
    "video_y = np.array(video_output_labels)\n",
    "print('audio_x shape: ', audio_x.shape)\n",
    "print('audio_y shape: ', audio_y.shape)\n",
    "print('video_x shape: ', video_x.shape)\n",
    "print('video_y shape: ', video_y.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1340 1205 2186 ... 2088 1070 1735]\n",
      "[ 970  347  819 ... 2049 1702 1913]\n"
     ]
    }
   ],
   "source": [
    "print(audio_y)\n",
    "print(video_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_split = int(audio_y.size * 0.2)\n",
    "audio_indices_val = np.where(audio_y < validation_split)\n",
    "video_indices_val = np.where(video_y < validation_split)\n",
    "audio_indices_train = np.where(audio_y >= validation_split)\n",
    "video_indices_train = np.where(video_y >= validation_split)\n",
    "validation_audio_x = audio_x[audio_indices_val]\n",
    "validation_audio_y = audio_y[audio_indices_val]\n",
    "validation_video_x = video_x[video_indices_val]\n",
    "validation_video_y = video_y[video_indices_val]\n",
    "training_audio_x = audio_x[audio_indices_train]\n",
    "training_audio_y = audio_y[audio_indices_train]\n",
    "training_video_x = video_x[video_indices_train]\n",
    "training_video_y = video_y[video_indices_train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- At this point the desired data should be loaded --- "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_batch(dataset_x, dataset_y, batch_size, labels=None):\n",
    "    if labels is None:\n",
    "        indices = list(np.random.randint(0, len(dataset_x), size=batch_size))\n",
    "    else:\n",
    "        indices=[]\n",
    "        for i in labels:\n",
    "            #print(i)\n",
    "            indices.append(np.where(dataset_y == i)[0][0])\n",
    "    # Recover what the entries for the batch are\n",
    "    batch_x = np.array([dataset_x[i] for i in indices])\n",
    "    batch_y = np.array([dataset_y[i] for i in indices])\n",
    "    \n",
    "    return batch_x, batch_y, indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_balanced_batch(dataset1_x, dataset1_y, dataset2_x, dataset2_y, batch_size):\n",
    "    b = int(batch_size/2)\n",
    "    indices = list(np.random.randint(0, len(dataset1_x), size=b))\n",
    "    labels = dataset1_y[indices]\n",
    "    matched1_x, matched1_y, _ = build_batch(dataset1_x, dataset1_y, b, labels)\n",
    "    matched2_x, matched2_y, _ = build_batch(dataset2_x, dataset2_y, b, labels)\n",
    "    random1_x, random1_y, _ = build_batch(dataset1_x, dataset1_y, b)    \n",
    "    random2_x, random2_y, _ = build_batch(dataset2_x, dataset2_y, b)   \n",
    "    x1 = np.concatenate([matched1_x, random1_x])   \n",
    "    y1 = np.concatenate([matched1_y, random1_y])  \n",
    "    x2 = np.concatenate([matched2_x, random2_x])   \n",
    "    y2 = np.concatenate([matched2_y, random2_y])  \n",
    "    return x1, y1, x2, y2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def contrastive_loss(embedded_batch_a, embedded_batch_b, a_labels, b_labels, margin):\n",
    "    y = tf.cast(tf.equal(a_labels, b_labels), tf.float32)\n",
    "    dist = tf.norm(embedded_batch_a - embedded_batch_b, axis=1)\n",
    "    #print(dist.shape)\n",
    "    loss = (y) * .5 * tf.square(dist) + (1-y) * .5 * tf.square(tf.maximum(0., margin - dist))\n",
    "    return y, tf.reduce_sum(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_graph(input_shape, labels_shape, \n",
    "                embed_size=128, learning_rate=0.001, \n",
    "                l1_reg=0.001, l2_reg=0.001, margin=50):\n",
    "    audio_inputs = tf.placeholder(shape=[None, input_shape[0], input_shape[1]], name=\"audio_inputs\", dtype=tf.float32)\n",
    "    video_inputs = tf.placeholder(shape=[None, input_shape[0], input_shape[1]], name=\"video_inputs\", dtype=tf.float32)\n",
    "    audio_labels = tf.placeholder(shape=[None], name=\"audio_labels\", dtype=tf.float32)\n",
    "    video_labels = tf.placeholder(shape=[None], name=\"video_labels\", dtype=tf.float32)\n",
    "\n",
    "    flattened_audio = tf.layers.flatten(audio_inputs)\n",
    "    flattened_video = tf.layers.flatten(video_inputs)\n",
    "    flattened_audio_labels = tf.layers.flatten(audio_labels)\n",
    "    flattened_video_labels = tf.layers.flatten(video_labels)\n",
    "\n",
    "    \n",
    "    #subnetwork = build_fc_net(inputs, embed_size, np.prod(video_input_shape), l1_reg, l2_reg)\n",
    "    weights = tf.trainable_variables()\n",
    "    reg_1 = tf.contrib.layers.l1_regularizer(scale=l1_reg)\n",
    "    reg_2 = tf.contrib.layers.l2_regularizer(scale=l2_reg)\n",
    "    audio_embed = build_fc_net(flattened_audio, embed_size, l1_reg, l2_reg)\n",
    "    video_embed = build_fc_net(flattened_video, embed_size, l1_reg, l2_reg, reuse=True)\n",
    "    weights = tf.trainable_variables()\n",
    "\n",
    "    #mse = tf.losses.mean_squared_error(audio_embed, video_embed) # only use loss if labels don't match?\n",
    "    matches, error = contrastive_loss(audio_embed, video_embed, flattened_audio_labels, flattened_video_labels, margin)\n",
    "    reg_penalty = tf.contrib.layers.apply_regularization(reg_1, weights) + tf.contrib.layers.apply_regularization(reg_2, weights)\n",
    "    loss = error + reg_penalty\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)\n",
    "    return (audio_inputs, audio_labels,\n",
    "            video_inputs, video_labels,\n",
    "            loss, optimizer, matches,\n",
    "            audio_embed, video_embed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "#audio input is a tf placeholder for the input audio features\n",
    "#video labels is a tf placeholder for the input video features\n",
    "#encode size is the desired size of the encoded vector (should be the same size as the video features)\n",
    "#l1 and l2 reg are the amount of weight to put on l1 and l2 regularizers for the loss\n",
    "\n",
    "def build_fc_net(input_data, embed_size, l1_reg=0.001, l2_reg=0.001, reuse=False, hidden_sizes=None):\n",
    "    net = build_fc_layers(input_data, embed_size, reuse, hidden_sizes)\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input layer should be the flattened inputs\n",
    "def build_fc_layers(input_layer, output_size, reuse=False, hidden_sizes=None):\n",
    "    if hidden_sizes is None:\n",
    "        hidden_sizes = [1024, 512, 256]\n",
    "    with tf.name_scope(\"model\"):\n",
    "        with tf.variable_scope(\"dense0\", reuse=reuse) as scope:\n",
    "            h1 = tf.layers.dense(inputs=input_layer, units=hidden_sizes[0], activation=tf.nn.tanh)\n",
    "            d1 = tf.layers.dropout(inputs=h1, rate=.3)\n",
    "        with tf.variable_scope(\"dense1\", reuse=reuse) as scope:\n",
    "            h2 = tf.layers.dense(inputs=d1, units=hidden_sizes[1], activation=tf.nn.tanh)\n",
    "            d2 = tf.layers.dropout(inputs=h2, rate=.3)            \n",
    "        with tf.variable_scope(\"dense2\", reuse=reuse) as scope:\n",
    "            h3 = tf.layers.dense(inputs=d2, units=hidden_sizes[2], activation=tf.nn.tanh)\n",
    "            d3 = tf.layers.dropout(inputs=h3, rate=.3)\n",
    "        with tf.variable_scope(\"dense3\", reuse=reuse) as scope:\n",
    "            raw_encode = tf.layers.dense(inputs=d3, units=output_size, activation=tf.nn.tanh)\n",
    "\n",
    "    return raw_encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(sess, audio_inputs, audio_labels, \n",
    "          video_inputs, video_labels, \n",
    "          audio_x, audio_y,\n",
    "          video_x, video_y,\n",
    "          loss, optimizer, \n",
    "          audio_embed, video_embed, \n",
    "          batch_size, cur_epoch, num_iters=1000):\n",
    "\n",
    "        for i in range(num_iters):\n",
    "            #audio_batch_input, audio_batch_label, video_batch_input, video_batch_label = build_balanced_batch(audio_x, audio_y, video_x, video_y, batch_size)\n",
    "            if i % 2 == 0:\n",
    "                audio_batch_input, audio_batch_label, _ = build_batch(audio_x, audio_y, batch_size)\n",
    "                video_batch_input, video_batch_label, _ = build_batch(video_x, video_y, batch_size)\n",
    "            else:\n",
    "                audio_batch_input, audio_batch_label, indices = build_batch(audio_x, audio_y, batch_size)\n",
    "                video_batch_input, video_batch_label, _ = build_batch(video_x, video_y, batch_size, audio_batch_label)\n",
    "            _, loss_val,ys = sess.run([optimizer, loss, matches], feed_dict={audio_inputs: audio_batch_input,\n",
    "                                                             video_inputs: video_batch_input,\n",
    "                                                             audio_labels: audio_batch_label,\n",
    "                                                             video_labels: video_batch_label})\n",
    "            if i % 100 == 0:\n",
    "                #train_summary_writer.add_scalar('train_loss', loss_val, cur_epoch * num_iters + i)\n",
    "                #print(ys)\n",
    "                print(\"Loss at iter \" + str(i) + \": \" + str(loss_val))\n",
    "                print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def k_nearest_neighbors(embedding, search_space, k):\n",
    "    embedding_stacked = np.repeat(embedding[np.newaxis,:], len(search_space), axis=0)\n",
    "    similarity = np.linalg.norm(embedding_stacked - search_space, axis=1)\n",
    "    #print(similarity.shape)\n",
    "    indices = np.argsort(similarity)[:k]\n",
    "    #print(indices)\n",
    "    return indices\n",
    "\n",
    "def validate(sess, audio_inputs, audio_labels, \n",
    "             video_inputs, video_labels, \n",
    "             audio_x, audio_y,\n",
    "             video_x, video_y,\n",
    "             loss, optimizer, \n",
    "             audio_embed, video_embed, \n",
    "             cur_epoch, batch_size=100, k=5):\n",
    "\n",
    "    audio_batch_input, audio_batch_label, indices = build_batch(audio_x, audio_y, batch_size)\n",
    "    #print(audio_batch_label)\n",
    "    video_batch_input, video_batch_label, _ = build_batch(video_x, video_y, batch_size, audio_batch_label)\n",
    "    #print(video_batch_label)\n",
    "    audio_embeddings = sess.run(audio_embed, feed_dict={audio_inputs: audio_batch_input,\n",
    "                                                          audio_labels: audio_batch_label})\n",
    "    video_embeddings = sess.run(video_embed, feed_dict={video_inputs: video_batch_input,\n",
    "                                                          video_labels: video_batch_label})\n",
    "    matches = 0\n",
    "    for i in range(len(audio_embeddings)):\n",
    "        a = audio_embeddings[i]\n",
    "        #if i%10 == 0:\n",
    "        #    print(a)\n",
    "        a_y = audio_batch_label[i]\n",
    "        ids = k_nearest_neighbors(a, video_embeddings, k)\n",
    "        #print(a_y)\n",
    "        #print(ids)\n",
    "        if a_y in video_batch_label[ids]:\n",
    "            matches += 1\n",
    "    random_val = k / len(audio_embeddings)\n",
    "    print(\"\")\n",
    "    print(\"Percent of matches found: \" + str(matches / len(audio_embeddings)))\n",
    "    print(\"Random Chance would be: \" + str(random_val))\n",
    "    #train_summary_writer.add_scalar('found_matches', matches/batch_size, cur_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 0\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 119639.79\n",
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-42-7c82e839f68b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     32\u001b[0m           \u001b[0mvideo_x\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvideo_y\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     33\u001b[0m           \u001b[0mloss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 34\u001b[1;33m           audio_embed, video_embed, batch, epoch)\n\u001b[0m\u001b[0;32m     35\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m     validate(sess, audio_inputs, audio_labels,\n",
      "\u001b[1;32m<ipython-input-40-6ff04fa4f358>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(sess, audio_inputs, audio_labels, video_inputs, video_labels, audio_x, audio_y, video_x, video_y, loss, optimizer, audio_embed, video_embed, batch_size, cur_epoch, num_iters)\u001b[0m\n\u001b[0;32m     18\u001b[0m                                                              \u001b[0mvideo_inputs\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mvideo_batch_input\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m                                                              \u001b[0maudio_labels\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0maudio_batch_label\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 20\u001b[1;33m                                                              video_labels: video_batch_label})\n\u001b[0m\u001b[0;32m     21\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m100\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m                 \u001b[1;31m#train_summary_writer.add_scalar('train_loss', loss_val, cur_epoch * num_iters + i)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    927\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    928\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 929\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    930\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    931\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1150\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1151\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1152\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1153\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1154\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1326\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1327\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[1;32m-> 1328\u001b[1;33m                            run_metadata)\n\u001b[0m\u001b[0;32m   1329\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1330\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1332\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1333\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1334\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1335\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1336\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1317\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1318\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m-> 1319\u001b[1;33m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[0;32m   1320\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1321\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[1;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[0;32m   1405\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[0;32m   1406\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1407\u001b[1;33m         run_metadata)\n\u001b[0m\u001b[0;32m   1408\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1409\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "train_log_dir = 'logs/tensorboard/train/log4'\n",
    "#train_summary_writer = SummaryWriter(train_log_dir)\n",
    "\n",
    "\n",
    "num_epochs = 100\n",
    "batch = 128\n",
    "embed_size = 64\n",
    "learning_rate = 0.005\n",
    "l1_reg = 0.0\n",
    "l2_reg = 0.001\n",
    "margin=5\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "(audio_inputs, audio_labels, \n",
    " video_inputs, video_labels, \n",
    " loss, optimizer, matches,\n",
    " audio_embed, video_embed) = build_graph(audio_x[0].shape, audio_y.shape[0], embed_size, learning_rate, l1_reg, l2_reg, margin)\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    print(\"\")\n",
    "    print(\"Epoch \" + str(epoch))\n",
    "    print(\"=============\")\n",
    "    print(\"\")\n",
    "\n",
    "    train(sess, audio_inputs, audio_labels,\n",
    "          video_inputs, video_labels,\n",
    "          audio_x, audio_y, \n",
    "          video_x, video_y, \n",
    "          loss, optimizer, \n",
    "          audio_embed, video_embed, batch, epoch)\n",
    "    \n",
    "    validate(sess, audio_inputs, audio_labels,\n",
    "          video_inputs, video_labels,\n",
    "          audio_x, audio_y, \n",
    "          video_x, video_y, \n",
    "          loss, optimizer, \n",
    "          audio_embed, video_embed, epoch, batch_size=validation_split)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validate(sess, audio_inputs, audio_labels,\n",
    "      video_inputs, video_labels,\n",
    "      audio_x, audio_y, \n",
    "      video_x, video_y, \n",
    "      loss, optimizer, \n",
    "      audio_embed, video_embed, epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_to_video_graph(input_shape, labels_shape, learning_rate=0.001, \n",
    "                l1_reg=0.001, l2_reg=0.001, margin=50):\n",
    "    audio_inputs = tf.placeholder(shape=[None, input_shape[0], input_shape[1]], name=\"audio_inputs\", dtype=tf.float32)\n",
    "    video_inputs = tf.placeholder(shape=[None, input_shape[0], input_shape[1]], name=\"video_inputs\", dtype=tf.float32)\n",
    "\n",
    "    flattened_audio = tf.layers.flatten(audio_inputs)\n",
    "    flattened_video = tf.layers.flatten(video_inputs)\n",
    "    \n",
    "    #subnetwork = build_fc_net(inputs, embed_size, np.prod(video_input_shape), l1_reg, l2_reg)\n",
    "    weights = tf.trainable_variables()\n",
    "    reg_1 = tf.contrib.layers.l1_regularizer(scale=l1_reg)\n",
    "    reg_2 = tf.contrib.layers.l2_regularizer(scale=l2_reg)\n",
    "    net = build_fc_net(flattened_audio, np.prod(input_shape), l1_reg, l2_reg, hidden_sizes=[1280, 1280, 1280])\n",
    "    weights = tf.trainable_variables()\n",
    "\n",
    "    error = tf.losses.mean_squared_error(net, flattened_video) # only use loss if labels don't match?\n",
    "    #matches, error = contrastive_loss(audio_embed, video_embed, flattened_audio_labels, flattened_video_labels, margin)\n",
    "    reg_penalty = tf.contrib.layers.apply_regularization(reg_1, weights) + tf.contrib.layers.apply_regularization(reg_2, weights)\n",
    "    loss = error + reg_penalty\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)\n",
    "    return (audio_inputs,\n",
    "            video_inputs,\n",
    "            loss, optimizer,\n",
    "            net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_a_to_v(sess, audio_inputs, \n",
    "          video_inputs, \n",
    "          audio_x, audio_y,\n",
    "          video_x, video_y,\n",
    "          loss, optimizer, \n",
    "          net, batch_size,\n",
    "          cur_epoch, num_iters=1000):\n",
    "\n",
    "        for i in range(num_iters):\n",
    "            audio_batch_input, audio_batch_label, indices = build_batch(audio_x, audio_y, batch_size)\n",
    "            video_batch_input, video_batch_label, _ = build_batch(video_x, video_y, batch_size, audio_batch_label)\n",
    "            _, loss_val = sess.run([optimizer, loss], feed_dict={audio_inputs: audio_batch_input,\n",
    "                                                             video_inputs: video_batch_input})\n",
    "            if i % 100 == 0:\n",
    "                #train_summary_writer.add_scalar('train_loss', loss_val, cur_epoch * num_iters + i)\n",
    "                #print(ys)\n",
    "                print(\"Loss at iter \" + str(i) + \": \" + str(loss_val))\n",
    "                print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_video_direct(sess, audio_inputs,\n",
    "             audio_x, audio_y,\n",
    "             video_x, video_y,\n",
    "             loss, optimizer,\n",
    "             net, cur_epoch, \n",
    "             batch_size=100, k=5):\n",
    "    \n",
    "    audio_embeddings = sess.run(net, feed_dict={audio_inputs: audio_x})\n",
    "    matches = 0\n",
    "    for i in range(len(audio_embeddings)):\n",
    "        a = audio_embeddings[i]\n",
    "        #if i%10 == 0:\n",
    "        #    print(a)\n",
    "        a_y = audio_y[i]\n",
    "        ids = k_nearest_neighbors(a, np.reshape(video_x, [video_x.shape[0], -1]), k)\n",
    "        #print(a_y)\n",
    "        #print(ids)\n",
    "        if a_y in video_y[ids]:\n",
    "            matches += 1\n",
    "    random_val = k / len(audio_embeddings)\n",
    "    print(\"\")\n",
    "    print(\"Percent of matches found: \" + str(matches / len(audio_embeddings)))\n",
    "    print(\"Random Chance would be: \" + str(random_val))\n",
    "    #train_summary_writer.add_scalar('found_matches', matches/batch_size, cur_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 0\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 3.6048768\n",
      "\n",
      "Loss at iter 100: 1.2104247\n",
      "\n",
      "Loss at iter 200: 0.92626446\n",
      "\n",
      "Loss at iter 300: 0.9222817\n",
      "\n",
      "Loss at iter 400: 0.9243196\n",
      "\n",
      "Loss at iter 500: 0.87257695\n",
      "\n",
      "Loss at iter 600: 0.90413153\n",
      "\n",
      "Loss at iter 700: 0.9001312\n",
      "\n",
      "Loss at iter 800: 0.90901756\n",
      "\n",
      "Loss at iter 900: 0.90472096\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 1\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.8975496\n",
      "\n",
      "Loss at iter 100: 0.8946828\n",
      "\n",
      "Loss at iter 200: 0.9204103\n",
      "\n",
      "Loss at iter 300: 0.90690285\n",
      "\n",
      "Loss at iter 400: 0.9165665\n",
      "\n",
      "Loss at iter 500: 0.90016055\n",
      "\n",
      "Loss at iter 600: 0.9207752\n",
      "\n",
      "Loss at iter 700: 0.9423184\n",
      "\n",
      "Loss at iter 800: 0.9048807\n",
      "\n",
      "Loss at iter 900: 0.9138212\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 2\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.9120704\n",
      "\n",
      "Loss at iter 100: 0.90556836\n",
      "\n",
      "Loss at iter 200: 0.87932324\n",
      "\n",
      "Loss at iter 300: 0.87622803\n",
      "\n",
      "Loss at iter 400: 0.91434586\n",
      "\n",
      "Loss at iter 500: 0.9000758\n",
      "\n",
      "Loss at iter 600: 0.892921\n",
      "\n",
      "Loss at iter 700: 0.9137151\n",
      "\n",
      "Loss at iter 800: 1.8715911\n",
      "\n",
      "Loss at iter 900: 1.5019093\n",
      "\n",
      "\n",
      "Percent of matches found: 0.002229157378510923\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 3\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 1.1892588\n",
      "\n",
      "Loss at iter 100: 0.9917535\n",
      "\n",
      "Loss at iter 200: 0.9363352\n",
      "\n",
      "Loss at iter 300: 0.89408886\n",
      "\n",
      "Loss at iter 400: 0.9226543\n",
      "\n",
      "Loss at iter 500: 0.9104738\n",
      "\n",
      "Loss at iter 600: 0.9112855\n",
      "\n",
      "Loss at iter 700: 0.9013564\n",
      "\n",
      "Loss at iter 800: 0.8707362\n",
      "\n",
      "Loss at iter 900: 0.918316\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 4\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.90389144\n",
      "\n",
      "Loss at iter 100: 0.89376485\n",
      "\n",
      "Loss at iter 200: 0.91688764\n",
      "\n",
      "Loss at iter 300: 0.9154543\n",
      "\n",
      "Loss at iter 400: 0.8914798\n",
      "\n",
      "Loss at iter 500: 0.92490995\n",
      "\n",
      "Loss at iter 600: 0.88373303\n",
      "\n",
      "Loss at iter 700: 0.9147583\n",
      "\n",
      "Loss at iter 800: 0.8902056\n",
      "\n",
      "Loss at iter 900: 0.904886\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 5\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.8688824\n",
      "\n",
      "Loss at iter 100: 0.9238195\n",
      "\n",
      "Loss at iter 200: 0.9183724\n",
      "\n",
      "Loss at iter 300: 0.88719445\n",
      "\n",
      "Loss at iter 400: 0.87208897\n",
      "\n",
      "Loss at iter 500: 0.9053292\n",
      "\n",
      "Loss at iter 600: 0.9178337\n",
      "\n",
      "Loss at iter 700: 0.8979189\n",
      "\n",
      "Loss at iter 800: 0.88170356\n",
      "\n",
      "Loss at iter 900: 0.8852965\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 6\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.8920359\n",
      "\n",
      "Loss at iter 100: 0.91815263\n",
      "\n",
      "Loss at iter 200: 0.89745396\n",
      "\n",
      "Loss at iter 300: 0.88189507\n",
      "\n",
      "Loss at iter 400: 0.9050654\n",
      "\n",
      "Loss at iter 500: 0.9001607\n",
      "\n",
      "Loss at iter 600: 1.5291188\n",
      "\n",
      "Loss at iter 700: 1.1510662\n",
      "\n",
      "Loss at iter 800: 1.013063\n",
      "\n",
      "Loss at iter 900: 1.0449387\n",
      "\n",
      "\n",
      "Percent of matches found: 0.002229157378510923\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 7\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.8943148\n",
      "\n",
      "Loss at iter 100: 0.885808\n",
      "\n",
      "Loss at iter 200: 0.913637\n",
      "\n",
      "Loss at iter 300: 0.93566716\n",
      "\n",
      "Loss at iter 400: 0.8896332\n",
      "\n",
      "Loss at iter 500: 0.9208055\n",
      "\n",
      "Loss at iter 600: 0.8850045\n",
      "\n",
      "Loss at iter 700: 0.889675\n",
      "\n",
      "Loss at iter 800: 0.9051667\n",
      "\n",
      "Loss at iter 900: 0.8968467\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 8\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.92508554\n",
      "\n",
      "Loss at iter 100: 0.9227623\n",
      "\n",
      "Loss at iter 200: 0.8646961\n",
      "\n",
      "Loss at iter 300: 0.8881646\n",
      "\n",
      "Loss at iter 400: 0.89450824\n",
      "\n",
      "Loss at iter 500: 0.8981218\n",
      "\n",
      "Loss at iter 600: 0.87868863\n",
      "\n",
      "Loss at iter 700: 0.87274873\n",
      "\n",
      "Loss at iter 800: 0.8901716\n",
      "\n",
      "Loss at iter 900: 0.92665505\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 9\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.9123804\n",
      "\n",
      "Loss at iter 100: 0.90201867\n",
      "\n",
      "Loss at iter 200: 0.89094335\n",
      "\n",
      "Loss at iter 300: 0.8974772\n",
      "\n",
      "Loss at iter 400: 0.8638337\n",
      "\n",
      "Loss at iter 500: 0.89211196\n",
      "\n",
      "Loss at iter 600: 0.9123163\n",
      "\n",
      "Loss at iter 700: 0.9108894\n",
      "\n",
      "Loss at iter 800: 0.8930985\n",
      "\n",
      "Loss at iter 900: 0.89107186\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 10\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 1.4509436\n",
      "\n",
      "Loss at iter 100: 1.2156959\n",
      "\n",
      "Loss at iter 200: 1.0189534\n",
      "\n",
      "Loss at iter 300: 0.9371548\n",
      "\n",
      "Loss at iter 400: 0.89848524\n",
      "\n",
      "Loss at iter 500: 0.92092603\n",
      "\n",
      "Loss at iter 600: 0.906392\n",
      "\n",
      "Loss at iter 700: 0.89543766\n",
      "\n",
      "Loss at iter 800: 0.86659753\n",
      "\n",
      "Loss at iter 900: 0.90006864\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 11\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.90676445\n",
      "\n",
      "Loss at iter 100: 0.9205802\n",
      "\n",
      "Loss at iter 200: 0.89374024\n",
      "\n",
      "Loss at iter 300: 0.92871183\n",
      "\n",
      "Loss at iter 400: 0.9159918\n",
      "\n",
      "Loss at iter 500: 0.9028379\n",
      "\n",
      "Loss at iter 600: 0.89994663\n",
      "\n",
      "Loss at iter 700: 0.87789375\n",
      "\n",
      "Loss at iter 800: 0.89964175\n",
      "\n",
      "Loss at iter 900: 0.8925531\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 12\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.89889157\n",
      "\n",
      "Loss at iter 100: 0.909585\n",
      "\n",
      "Loss at iter 200: 0.9114496\n",
      "\n",
      "Loss at iter 300: 0.9230855\n",
      "\n",
      "Loss at iter 400: 0.8855464\n",
      "\n",
      "Loss at iter 500: 0.89206415\n",
      "\n",
      "Loss at iter 600: 0.87691766\n",
      "\n",
      "Loss at iter 700: 0.89278275\n",
      "\n",
      "Loss at iter 800: 0.89520043\n",
      "\n",
      "Loss at iter 900: 0.89298075\n",
      "\n",
      "\n",
      "Percent of matches found: 0.002229157378510923\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 13\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.91499346\n",
      "\n",
      "Loss at iter 100: 0.92754406\n",
      "\n",
      "Loss at iter 200: 0.8940411\n",
      "\n",
      "Loss at iter 300: 1.1415253\n",
      "\n",
      "Loss at iter 400: 1.6492585\n",
      "\n",
      "Loss at iter 500: 1.0708804\n",
      "\n",
      "Loss at iter 600: 0.9391644\n",
      "\n",
      "Loss at iter 700: 0.9046156\n",
      "\n",
      "Loss at iter 800: 0.91072387\n",
      "\n",
      "Loss at iter 900: 0.9147859\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 14\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.89128256\n",
      "\n",
      "Loss at iter 100: 0.90465075\n",
      "\n",
      "Loss at iter 200: 0.9034962\n",
      "\n",
      "Loss at iter 300: 0.9170957\n",
      "\n",
      "Loss at iter 400: 0.9211259\n",
      "\n",
      "Loss at iter 500: 0.8902115\n",
      "\n",
      "Loss at iter 600: 0.9098964\n",
      "\n",
      "Loss at iter 700: 0.8977543\n",
      "\n",
      "Loss at iter 800: 0.8750811\n",
      "\n",
      "Loss at iter 900: 0.9131431\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 15\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.91485095\n",
      "\n",
      "Loss at iter 100: 0.90939677\n",
      "\n",
      "Loss at iter 200: 0.89039814\n",
      "\n",
      "Loss at iter 300: 0.902812\n",
      "\n",
      "Loss at iter 400: 0.8932781\n",
      "\n",
      "Loss at iter 500: 0.8887769\n",
      "\n",
      "Loss at iter 600: 0.9170657\n",
      "\n",
      "Loss at iter 700: 0.8912302\n",
      "\n",
      "Loss at iter 800: 0.888026\n",
      "\n",
      "Loss at iter 900: 0.8945506\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 16\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.8948407\n",
      "\n",
      "Loss at iter 100: 0.8983825\n",
      "\n",
      "Loss at iter 200: 0.9040241\n",
      "\n",
      "Loss at iter 300: 0.8948908\n",
      "\n",
      "Loss at iter 400: 0.8705113\n",
      "\n",
      "Loss at iter 500: 0.892637\n",
      "\n",
      "Loss at iter 600: 0.879956\n",
      "\n",
      "Loss at iter 700: 0.8931841\n",
      "\n",
      "Loss at iter 800: 0.88912797\n",
      "\n",
      "Loss at iter 900: 0.9006353\n",
      "\n",
      "\n",
      "Percent of matches found: 0.0026749888542131075\n",
      "Random Chance would be: 0.002229157378510923\n",
      "\n",
      "Epoch 17\n",
      "=============\n",
      "\n",
      "Loss at iter 0: 0.9023228\n",
      "\n",
      "Loss at iter 100: 0.90770173\n",
      "\n",
      "Loss at iter 200: 1.2544214\n",
      "\n",
      "Loss at iter 300: 1.205086\n",
      "\n",
      "Loss at iter 400: 1.0167648\n",
      "\n",
      "Loss at iter 500: 0.940628\n",
      "\n",
      "Loss at iter 600: 0.8975816\n",
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-86-8678cbdf65a1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     29\u001b[0m           \u001b[0mvideo_x\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvideo_y\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     30\u001b[0m           \u001b[0mloss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 31\u001b[1;33m           net, batch, epoch)\n\u001b[0m\u001b[0;32m     32\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     33\u001b[0m     validate_video_direct(sess, audio_inputs,\n",
      "\u001b[1;32m<ipython-input-84-bfb6f9551e13>\u001b[0m in \u001b[0;36mtrain_a_to_v\u001b[1;34m(sess, audio_inputs, video_inputs, audio_x, audio_y, video_x, video_y, loss, optimizer, net, batch_size, cur_epoch, num_iters)\u001b[0m\n\u001b[0;32m     11\u001b[0m             \u001b[0mvideo_batch_input\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvideo_batch_label\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbuild_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvideo_x\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvideo_y\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maudio_batch_label\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m             _, loss_val = sess.run([optimizer, loss], feed_dict={audio_inputs: audio_batch_input,\n\u001b[1;32m---> 13\u001b[1;33m                                                              video_inputs: video_batch_input})\n\u001b[0m\u001b[0;32m     14\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m100\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m                 \u001b[1;31m#train_summary_writer.add_scalar('train_loss', loss_val, cur_epoch * num_iters + i)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    927\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    928\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 929\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    930\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    931\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1150\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1151\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1152\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1153\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1154\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1326\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1327\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[1;32m-> 1328\u001b[1;33m                            run_metadata)\n\u001b[0m\u001b[0;32m   1329\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1330\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1332\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1333\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1334\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1335\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1336\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1317\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1318\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m-> 1319\u001b[1;33m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[0;32m   1320\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1321\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[1;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[0;32m   1405\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[0;32m   1406\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1407\u001b[1;33m         run_metadata)\n\u001b[0m\u001b[0;32m   1408\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1409\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "train_log_dir = 'logs/tensorboard/train/log4'\n",
    "#train_summary_writer = SummaryWriter(train_log_dir)\n",
    "\n",
    "\n",
    "num_epochs = 200\n",
    "batch = 64\n",
    "learning_rate = 0.001\n",
    "l1_reg = 0.0\n",
    "l2_reg = 0.001\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "(audio_inputs, video_inputs, \n",
    " loss, optimizer,\n",
    " net) = build_to_video_graph(audio_x[0].shape, audio_y.shape[0], learning_rate, l1_reg, l2_reg, margin)\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    print(\"\")\n",
    "    print(\"Epoch \" + str(epoch))\n",
    "    print(\"=============\")\n",
    "    print(\"\")\n",
    "\n",
    "    train_a_to_v(sess, audio_inputs,\n",
    "          video_inputs,\n",
    "          audio_x, audio_y, \n",
    "          video_x, video_y, \n",
    "          loss, optimizer, \n",
    "          net, batch, epoch)\n",
    "\n",
    "    validate_video_direct(sess, audio_inputs,\n",
    "                         audio_x, audio_y,\n",
    "                         video_x, video_y, \n",
    "                         loss, optimizer,\n",
    "                         net, epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_lstm_graph(input_shape, labels_shape, \n",
    "                embed_size=128, learning_rate=0.001, \n",
    "                l1_reg=0.001, l2_reg=0.001, margin=50):\n",
    "    audio_inputs = tf.placeholder(shape=[None, input_shape[0], input_shape[1]], name=\"audio_inputs\", dtype=tf.float32)\n",
    "    video_inputs = tf.placeholder(shape=[None, input_shape[0], input_shape[1]], name=\"video_inputs\", dtype=tf.float32)\n",
    "    audio_labels = tf.placeholder(shape=[None], name=\"audio_labels\", dtype=tf.float32)\n",
    "    video_labels = tf.placeholder(shape=[None], name=\"video_labels\", dtype=tf.float32)\n",
    "    \n",
    "    split_audio = tf.unstack(audio_inputs, axis=1)\n",
    "    split_video = tf.unstack(video_inputs, axis=1)\n",
    "    flattened_audio_labels = tf.layers.flatten(audio_labels)\n",
    "    flattened_video_labels = tf.layers.flatten(video_labels)\n",
    "\n",
    "    #subnetwork = build_fc_net(inputs, embed_size, np.prod(video_input_shape), l1_reg, l2_reg)\n",
    "    weights = tf.trainable_variables()\n",
    "    reg_1 = tf.contrib.layers.l1_regularizer(scale=l1_reg)\n",
    "    reg_2 = tf.contrib.layers.l2_regularizer(scale=l2_reg)\n",
    "    lstm = tf.nn.rnn_cell.BasicLSTMCell(512)\n",
    "    lstm_audio, audio_states = tf.nn.static_rnn(lstm, split_audio, dtype=tf.float32)\n",
    "    lstm_video, video_states = tf.nn.static_rnn(lstm, split_video, dtype=tf.float32)\n",
    "    audio_embed = build_fc_net(lstm_audio[-1], embed_size, l1_reg, l2_reg)\n",
    "    video_embed = build_fc_net(lstm_video[-1], embed_size, l1_reg, l2_reg, reuse=True)\n",
    "    weights = tf.trainable_variables()\n",
    "\n",
    "    #mse = tf.losses.mean_squared_error(audio_embed, video_embed) # only use loss if labels don't match?\n",
    "    matches, error = contrastive_loss(audio_embed, video_embed, flattened_audio_labels, flattened_video_labels, margin)\n",
    "    reg_penalty = tf.contrib.layers.apply_regularization(reg_1, weights) + tf.contrib.layers.apply_regularization(reg_2, weights)\n",
    "    loss = error + reg_penalty\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)\n",
    "    return (audio_inputs, audio_labels,\n",
    "            video_inputs, video_labels,\n",
    "            loss, optimizer, matches,\n",
    "            audio_embed, video_embed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36mget_attr\u001b[1;34m(self, name)\u001b[0m\n\u001b[0;32m   2408\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0mc_api_util\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtf_buffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mbuf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2409\u001b[1;33m         \u001b[0mc_api\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_OperationGetAttrValueProto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_c_op\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbuf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2410\u001b[0m         \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mc_api\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbuf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInvalidArgumentError\u001b[0m: Operation 'rnn/basic_lstm_cell/Add_30' has no attr named '_XlaCompile'.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_impl.py\u001b[0m in \u001b[0;36m_MaybeCompile\u001b[1;34m(scope, op, func, grad_fn)\u001b[0m\n\u001b[0;32m    414\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 415\u001b[1;33m       \u001b[0mxla_compile\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_attr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"_XlaCompile\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    416\u001b[0m       xla_separate_compiled_gradients = op.get_attr(\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36mget_attr\u001b[1;34m(self, name)\u001b[0m\n\u001b[0;32m   2412\u001b[0m       \u001b[1;31m# Convert to ValueError for backwards compatibility.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2413\u001b[1;33m       \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2414\u001b[0m     \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mattr_value_pb2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mAttrValue\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Operation 'rnn/basic_lstm_cell/Add_30' has no attr named '_XlaCompile'.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-147-c73059d7ecc5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     16\u001b[0m  \u001b[0mvideo_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvideo_labels\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m  \u001b[0mloss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmatches\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 18\u001b[1;33m  audio_embed, video_embed) = build_lstm_graph(audio_x[0].shape, audio_y.shape[0], embed_size, learning_rate, l1_reg, l2_reg, margin)\n\u001b[0m\u001b[0;32m     19\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m \u001b[0msess\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-146-5dc5cfe4074a>\u001b[0m in \u001b[0;36mbuild_lstm_graph\u001b[1;34m(input_shape, labels_shape, embed_size, learning_rate, l1_reg, l2_reg, margin)\u001b[0m\n\u001b[0;32m     27\u001b[0m     \u001b[0mreg_penalty\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply_regularization\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mreg_1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply_regularization\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mreg_2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m     \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0merror\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mreg_penalty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 29\u001b[1;33m     \u001b[0moptimizer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mAdamOptimizer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearning_rate\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mminimize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     30\u001b[0m     return (audio_inputs, audio_labels,\n\u001b[0;32m     31\u001b[0m             \u001b[0mvideo_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvideo_labels\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\training\\optimizer.py\u001b[0m in \u001b[0;36mminimize\u001b[1;34m(self, loss, global_step, var_list, gate_gradients, aggregation_method, colocate_gradients_with_ops, name, grad_loss)\u001b[0m\n\u001b[0;32m    401\u001b[0m         \u001b[0maggregation_method\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0maggregation_method\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    402\u001b[0m         \u001b[0mcolocate_gradients_with_ops\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcolocate_gradients_with_ops\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 403\u001b[1;33m         grad_loss=grad_loss)\n\u001b[0m\u001b[0;32m    404\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    405\u001b[0m     \u001b[0mvars_with_grad\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mv\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mg\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mv\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mgrads_and_vars\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mg\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\training\\optimizer.py\u001b[0m in \u001b[0;36mcompute_gradients\u001b[1;34m(self, loss, var_list, gate_gradients, aggregation_method, colocate_gradients_with_ops, grad_loss)\u001b[0m\n\u001b[0;32m    510\u001b[0m         \u001b[0mgate_gradients\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgate_gradients\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mOptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mGATE_OP\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    511\u001b[0m         \u001b[0maggregation_method\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0maggregation_method\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 512\u001b[1;33m         colocate_gradients_with_ops=colocate_gradients_with_ops)\n\u001b[0m\u001b[0;32m    513\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mgate_gradients\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mOptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mGATE_GRAPH\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    514\u001b[0m       \u001b[0mgrads\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcontrol_flow_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgrads\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_impl.py\u001b[0m in \u001b[0;36mgradients\u001b[1;34m(ys, xs, grad_ys, name, colocate_gradients_with_ops, gate_gradients, aggregation_method, stop_gradients, unconnected_gradients)\u001b[0m\n\u001b[0;32m    662\u001b[0m     return _GradientsHelper(ys, xs, grad_ys, name, colocate_gradients_with_ops,\n\u001b[0;32m    663\u001b[0m                             \u001b[0mgate_gradients\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maggregation_method\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstop_gradients\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 664\u001b[1;33m                             unconnected_gradients)\n\u001b[0m\u001b[0;32m    665\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    666\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_impl.py\u001b[0m in \u001b[0;36m_GradientsHelper\u001b[1;34m(ys, xs, grad_ys, name, colocate_gradients_with_ops, gate_gradients, aggregation_method, stop_gradients, unconnected_gradients, src_graph)\u001b[0m\n\u001b[0;32m    963\u001b[0m                 \u001b[1;31m# functions.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    964\u001b[0m                 in_grads = _MaybeCompile(grad_scope, op, func_call,\n\u001b[1;32m--> 965\u001b[1;33m                                          lambda: grad_fn(op, *out_grads))\n\u001b[0m\u001b[0;32m    966\u001b[0m               \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    967\u001b[0m                 \u001b[1;31m# For function call ops, we add a 'SymbolicGradient'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_impl.py\u001b[0m in \u001b[0;36m_MaybeCompile\u001b[1;34m(scope, op, func, grad_fn)\u001b[0m\n\u001b[0;32m    418\u001b[0m       \u001b[0mxla_scope\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_attr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"_XlaScope\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    419\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 420\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mgrad_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# Exit early\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    421\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    422\u001b[0m   \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mxla_compile\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_impl.py\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    963\u001b[0m                 \u001b[1;31m# functions.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    964\u001b[0m                 in_grads = _MaybeCompile(grad_scope, op, func_call,\n\u001b[1;32m--> 965\u001b[1;33m                                          lambda: grad_fn(op, *out_grads))\n\u001b[0m\u001b[0;32m    966\u001b[0m               \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    967\u001b[0m                 \u001b[1;31m# For function call ops, we add a 'SymbolicGradient'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py\u001b[0m in \u001b[0;36m_AddGrad\u001b[1;34m(op, grad)\u001b[0m\n\u001b[0;32m    907\u001b[0m   \u001b[0msy\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    908\u001b[0m   \u001b[0mrx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mry\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgen_array_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbroadcast_gradient_args\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msy\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 909\u001b[1;33m   return (array_ops.reshape(math_ops.reduce_sum(grad, rx), sx),\n\u001b[0m\u001b[0;32m    910\u001b[0m           array_ops.reshape(math_ops.reduce_sum(grad, ry), sy))\n\u001b[0;32m    911\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\gen_array_ops.py\u001b[0m in \u001b[0;36mreshape\u001b[1;34m(tensor, shape, name)\u001b[0m\n\u001b[0;32m   7176\u001b[0m   \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   7177\u001b[0m     _, _, _op = _op_def_lib._apply_op_helper(\n\u001b[1;32m-> 7178\u001b[1;33m         \"Reshape\", tensor=tensor, shape=shape, name=name)\n\u001b[0m\u001b[0;32m   7179\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mTypeError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   7180\u001b[0m     result = _dispatch.dispatch(\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\u001b[0m in \u001b[0;36m_apply_op_helper\u001b[1;34m(self, op_type_name, name, **keywords)\u001b[0m\n\u001b[0;32m    786\u001b[0m         op = g.create_op(op_type_name, inputs, output_types, name=scope,\n\u001b[0;32m    787\u001b[0m                          \u001b[0minput_types\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minput_types\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mattr_protos\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 788\u001b[1;33m                          op_def=op_def)\n\u001b[0m\u001b[0;32m    789\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0moutput_structure\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop_def\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_stateful\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    790\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\util\\deprecation.py\u001b[0m in \u001b[0;36mnew_func\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    505\u001b[0m                 \u001b[1;34m'in a future version'\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mdate\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m'after %s'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mdate\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    506\u001b[0m                 instructions)\n\u001b[1;32m--> 507\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    508\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    509\u001b[0m     doc = _add_deprecated_arg_notice_to_docstring(\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36mcreate_op\u001b[1;34m(***failed resolving arguments***)\u001b[0m\n\u001b[0;32m   3298\u001b[0m           \u001b[0minput_types\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minput_types\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3299\u001b[0m           \u001b[0moriginal_op\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_default_original_op\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3300\u001b[1;33m           op_def=op_def)\n\u001b[0m\u001b[0;32m   3301\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_create_op_helper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mret\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcompute_device\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcompute_device\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3302\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mret\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, node_def, g, inputs, output_types, control_inputs, input_types, original_op, op_def)\u001b[0m\n\u001b[0;32m   1821\u001b[0m           op_def, inputs, node_def.attr)\n\u001b[0;32m   1822\u001b[0m       self._c_op = _create_c_op(self._graph, node_def, grouped_inputs,\n\u001b[1;32m-> 1823\u001b[1;33m                                 control_input_ops)\n\u001b[0m\u001b[0;32m   1824\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1825\u001b[0m     \u001b[1;31m# Initialize self._outputs.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36m_create_c_op\u001b[1;34m(graph, node_def, inputs, control_inputs)\u001b[0m\n\u001b[0;32m   1634\u001b[0m   op_desc = c_api.TF_NewOperation(graph._c_graph,\n\u001b[0;32m   1635\u001b[0m                                   \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_str\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mop\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1636\u001b[1;33m                                   compat.as_str(node_def.name))\n\u001b[0m\u001b[0;32m   1637\u001b[0m   \u001b[1;32mif\u001b[0m \u001b[0mnode_def\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1638\u001b[0m     \u001b[0mc_api\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_SetDevice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mop_desc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_str\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "train_log_dir = 'logs/tensorboard/train/log4'\n",
    "#train_summary_writer = SummaryWriter(train_log_dir)\n",
    "\n",
    "\n",
    "num_epochs = 100\n",
    "batch = 64\n",
    "embed_size = 128\n",
    "learning_rate = 0.005\n",
    "l1_reg = 0.0\n",
    "l2_reg = 0.001\n",
    "margin=5\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "(audio_inputs, audio_labels, \n",
    " video_inputs, video_labels, \n",
    " loss, optimizer, matches,\n",
    " audio_embed, video_embed) = build_lstm_graph(audio_x[0].shape, audio_y.shape[0], embed_size, learning_rate, l1_reg, l2_reg, margin)\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    print(\"\")\n",
    "    print(\"Epoch \" + str(epoch))\n",
    "    print(\"=============\")\n",
    "    print(\"\")\n",
    "\n",
    "    train(sess, audio_inputs, audio_labels,\n",
    "          video_inputs, video_labels,\n",
    "          audio_x, audio_y, \n",
    "          video_x, video_y, \n",
    "          loss, optimizer, \n",
    "          audio_embed, video_embed, batch, epoch)\n",
    "    \n",
    "    validate(sess, audio_inputs, audio_labels,\n",
    "          video_inputs, video_labels,\n",
    "          audio_x, audio_y, \n",
    "          video_x, video_y, \n",
    "          loss, optimizer, \n",
    "          audio_embed, video_embed, epoch, batch_size=validation_split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}