From 15cf6a74e1f00b5a8c4a91fb7aa83ae6277fa880 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Fri, 28 May 2021 23:02:26 -0400 Subject: [PATCH 1/6] adding 7.1 jupyter notebook --- 7.1-build-a-concatenate-network.ipynb | 159 ++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 7.1-build-a-concatenate-network.ipynb diff --git a/7.1-build-a-concatenate-network.ipynb b/7.1-build-a-concatenate-network.ipynb new file mode 100644 index 0000000000..478ff793b8 --- /dev/null +++ b/7.1-build-a-concatenate-network.ipynb @@ -0,0 +1,159 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# two inputs, one output model" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"model_8\"\n", + "__________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + "text (InputLayer) [(None, None)] 0 \n", + "__________________________________________________________________________________________________\n", + "question (InputLayer) [(None, None)] 0 \n", + "__________________________________________________________________________________________________\n", + "embedding_23 (Embedding) (None, None, 10000) 640000 text[0][0] \n", + "__________________________________________________________________________________________________\n", + "embedding_24 (Embedding) (None, None, 10000) 320000 question[0][0] \n", + "__________________________________________________________________________________________________\n", + "lstm_23 (LSTM) (None, 32) 1284224 embedding_23[0][0] \n", + "__________________________________________________________________________________________________\n", + "lstm_24 (LSTM) (None, 32) 1284224 embedding_24[0][0] \n", + "__________________________________________________________________________________________________\n", + "concatenate_10 (Concatenate) (None, 64) 0 lstm_23[0][0] \n", + " lstm_24[0][0] \n", + "__________________________________________________________________________________________________\n", + "dense_9 (Dense) (None, 500) 32500 concatenate_10[0][0] \n", + "==================================================================================================\n", + "Total params: 3,560,948\n", + "Trainable params: 3,560,948\n", + "Non-trainable params: 0\n", + "__________________________________________________________________________________________________\n", + "None\n" + ] + } + ], + "source": [ + "from keras.models import Model\n", + "from keras import layers\n", + "from keras import Input\n", + "\n", + "text_vocabulary_size= 10000\n", + "question_vocabulary_size = 10000\n", + "answer_vocabulary_size = 500\n", + "\n", + "\n", + "text_input = Input(shape=(None, ), dtype = 'int32', name ='text')\n", + "embedded_text = layers.Embedding(64, text_vocabulary_size)(text_input)\n", + "encoded_text = layers.LSTM(32)(embedded_text)\n", + "\n", + "question_input = Input(shape=(None,), dtype = 'int32', name = 'question')\n", + "embedded_question = layers.Embedding(32, question_vocabulary_size)(question_input)\n", + "encoded_question = layers.LSTM(16)(embedded_question)\n", + "\n", + "# concatenate two inputs and give output\n", + "concatenated = layers.concatenate([encoded_text, encoded_question], axis=-1)\n", + "answer = layers.Dense(answer_vocabulary_size, activation='softmax')(concatenated)\n", + "\n", + "model = Model([text_input, question_input], answer)\n", + "model.compile(optimizer = 'rmsprop', loss='categorical_crossentropy', metrics=['acc'])\n", + "print(model.summary())" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n" + ] + }, + { + "ename": "InvalidArgumentError", + "evalue": " indices[126,0] = 9504 is not in [0, 32)\n\t [[node model_8/embedding_24/embedding_lookup (defined at :13) ]] [Op:__inference_train_function_44332]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node model_8/embedding_24/embedding_lookup:\n model_8/embedding_24/embedding_lookup/42066 (defined at /Users/chaowu/opt/anaconda3/lib/python3.8/contextlib.py:113)\n\nFunction call stack:\ntrain_function\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mInvalidArgumentError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquestion\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0manswers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m128\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 14\u001b[0m \u001b[0;31m#model.fit({\"text\":text, \"question\":question}, answers, epochs=10, batch_size=128)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m 1098\u001b[0m _r=1):\n\u001b[1;32m 1099\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1100\u001b[0;31m \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1101\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1102\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 826\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 827\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 828\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 829\u001b[0m \u001b[0mcompiler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"xla\"\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_experimental_compile\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m\"nonXla\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 830\u001b[0m \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 853\u001b[0m \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 854\u001b[0m \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 855\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 856\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 857\u001b[0m \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 2940\u001b[0m (graph_function,\n\u001b[1;32m 2941\u001b[0m filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[0;32m-> 2942\u001b[0;31m return graph_function._call_flat(\n\u001b[0m\u001b[1;32m 2943\u001b[0m filtered_flat_args, captured_inputs=graph_function.captured_inputs) # pylint: disable=protected-access\n\u001b[1;32m 2944\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m 1916\u001b[0m and executing_eagerly):\n\u001b[1;32m 1917\u001b[0m \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1918\u001b[0;31m return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[1;32m 1919\u001b[0m ctx, args, cancellation_manager=cancellation_manager))\n\u001b[1;32m 1920\u001b[0m forward_backward = self._select_forward_and_backward_functions(\n", + "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m 553\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 554\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 555\u001b[0;31m outputs = execute.execute(\n\u001b[0m\u001b[1;32m 556\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 557\u001b[0m \u001b[0mnum_outputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[1;32m 60\u001b[0m inputs, attrs, num_outputs)\n\u001b[1;32m 61\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mInvalidArgumentError\u001b[0m: indices[126,0] = 9504 is not in [0, 32)\n\t [[node model_8/embedding_24/embedding_lookup (defined at :13) ]] [Op:__inference_train_function_44332]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node model_8/embedding_24/embedding_lookup:\n model_8/embedding_24/embedding_lookup/42066 (defined at /Users/chaowu/opt/anaconda3/lib/python3.8/contextlib.py:113)\n\nFunction call stack:\ntrain_function\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "\n", + "num_samples = 1000\n", + "max_length = 100\n", + "\n", + "text = np.random.randint(1, text_vocabulary_size, size = (num_samples, max_length))\n", + "\n", + "question = np.random.randint(1, question_vocabulary_size, size = (num_samples, max_length))\n", + "\n", + "answers = np.random.randint(0,1, size = (num_samples, answer_vocabulary_size))\n", + "\n", + "\n", + "model.fit([text, question], answers, epochs = 10, batch_size = 128)\n", + "#model.fit({\"text\":text, \"question\":question}, answers, epochs=10, batch_size=128)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 2fc65c0dc8edda9b83dcddc929400ec434d0ad36 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Fri, 28 May 2021 23:21:52 -0400 Subject: [PATCH 2/6] adding two examples from Chapter 7 --- 7.1.1-introduction-to-funcational-API.ipynb | 117 ++++++++++++++++++ ...=> 7.1.2-build-a-concatenate-network.ipynb | 0 2 files changed, 117 insertions(+) create mode 100644 7.1.1-introduction-to-funcational-API.ipynb rename 7.1-build-a-concatenate-network.ipynb => 7.1.2-build-a-concatenate-network.ipynb (100%) diff --git a/7.1.1-introduction-to-funcational-API.ipynb b/7.1.1-introduction-to-funcational-API.ipynb new file mode 100644 index 0000000000..c8b7aefa87 --- /dev/null +++ b/7.1.1-introduction-to-funcational-API.ipynb @@ -0,0 +1,117 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"sequential\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "dense (Dense) (None, 32) 2080 \n", + "_________________________________________________________________\n", + "dense_1 (Dense) (None, 32) 1056 \n", + "_________________________________________________________________\n", + "dense_2 (Dense) (None, 10) 330 \n", + "=================================================================\n", + "Total params: 3,466\n", + "Trainable params: 3,466\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n", + "Model: \"model\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "input_1 (InputLayer) [(None, 64)] 0 \n", + "_________________________________________________________________\n", + "dense_3 (Dense) (None, 32) 2080 \n", + "_________________________________________________________________\n", + "dense_4 (Dense) (None, 32) 1056 \n", + "_________________________________________________________________\n", + "dense_5 (Dense) (None, 10) 330 \n", + "=================================================================\n", + "Total params: 3,466\n", + "Trainable params: 3,466\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "from keras.models import Sequential, Model \n", + "from keras import layers\n", + "from keras import Input\n", + "seq_model = Sequential()\n", + "seq_model.add(layers.Dense(32, activation='relu', input_shape=(64,))) \n", + "seq_model.add(layers.Dense(32, activation='relu')) \n", + "seq_model.add(layers.Dense(10, activation='softmax'))\n", + "seq_model.summary()\n", + "\n", + "input_tensor = Input(shape=(64,))\n", + "x = layers.Dense(32, activation='relu')(input_tensor)\n", + "x = layers.Dense(32, activation='relu')(x)\n", + "output_tensor = layers.Dense(10, activation='softmax')(x)\n", + "model = Model(input_tensor, output_tensor) \n", + "model.summary() " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "invalid syntax (, line 6)", + "output_type": "error", + "traceback": [ + "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m6\u001b[0m\n\u001b[0;31m Generates dummy Numpy data to train on\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" + ] + } + ], + "source": [ + "model.compile(optimizer='rmsprop', loss='categorical_crossentropy')\n", + "\n", + "import numpy as np\n", + "x_train = np.random.random((1000, 64)) \n", + "y_train = np.random.random((1000, 10))\n", + "\n", + "model.fit(x_train, y_train, epochs=10, batch_size=128)\n", + "score = model.evaluate(x_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/7.1-build-a-concatenate-network.ipynb b/7.1.2-build-a-concatenate-network.ipynb similarity index 100% rename from 7.1-build-a-concatenate-network.ipynb rename to 7.1.2-build-a-concatenate-network.ipynb From 4126a30d1a460e327147676b051911ea2c6d892f Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Sat, 29 May 2021 00:10:17 -0400 Subject: [PATCH 3/6] fixing the error in 7.1.2 Embedding lines(parameter orders are wrong --- 7.1.2-build-a-concatenate-network.ipynb | 100 +++++++++++++++--------- 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/7.1.2-build-a-concatenate-network.ipynb b/7.1.2-build-a-concatenate-network.ipynb index 478ff793b8..4d860f1232 100644 --- a/7.1.2-build-a-concatenate-network.ipynb +++ b/7.1.2-build-a-concatenate-network.ipynb @@ -9,14 +9,14 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 34, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Model: \"model_8\"\n", + "Model: \"model_9\"\n", "__________________________________________________________________________________________________\n", "Layer (type) Output Shape Param # Connected to \n", "==================================================================================================\n", @@ -24,21 +24,21 @@ "__________________________________________________________________________________________________\n", "question (InputLayer) [(None, None)] 0 \n", "__________________________________________________________________________________________________\n", - "embedding_23 (Embedding) (None, None, 10000) 640000 text[0][0] \n", + "embedding_20 (Embedding) (None, None, 64) 640000 text[0][0] \n", "__________________________________________________________________________________________________\n", - "embedding_24 (Embedding) (None, None, 10000) 320000 question[0][0] \n", + "embedding_21 (Embedding) (None, None, 32) 320000 question[0][0] \n", "__________________________________________________________________________________________________\n", - "lstm_23 (LSTM) (None, 32) 1284224 embedding_23[0][0] \n", + "lstm_20 (LSTM) (None, 32) 12416 embedding_20[0][0] \n", "__________________________________________________________________________________________________\n", - "lstm_24 (LSTM) (None, 32) 1284224 embedding_24[0][0] \n", + "lstm_21 (LSTM) (None, 16) 3136 embedding_21[0][0] \n", "__________________________________________________________________________________________________\n", - "concatenate_10 (Concatenate) (None, 64) 0 lstm_23[0][0] \n", - " lstm_24[0][0] \n", + "concatenate_8 (Concatenate) (None, 48) 0 lstm_20[0][0] \n", + " lstm_21[0][0] \n", "__________________________________________________________________________________________________\n", - "dense_9 (Dense) (None, 500) 32500 concatenate_10[0][0] \n", + "dense_8 (Dense) (None, 500) 24500 concatenate_8[0][0] \n", "==================================================================================================\n", - "Total params: 3,560,948\n", - "Trainable params: 3,560,948\n", + "Total params: 1,000,052\n", + "Trainable params: 1,000,052\n", "Non-trainable params: 0\n", "__________________________________________________________________________________________________\n", "None\n" @@ -50,57 +50,68 @@ "from keras import layers\n", "from keras import Input\n", "\n", - "text_vocabulary_size= 10000\n", + "text_vocabulary_size = 10000\n", "question_vocabulary_size = 10000\n", "answer_vocabulary_size = 500\n", "\n", - "\n", - "text_input = Input(shape=(None, ), dtype = 'int32', name ='text')\n", - "embedded_text = layers.Embedding(64, text_vocabulary_size)(text_input)\n", - "encoded_text = layers.LSTM(32)(embedded_text)\n", + "text_input = Input(shape=(None,), dtype = 'int32', name ='text')\n", + "embedded_text = layers.Embedding(text_vocabulary_size,64)(text_input)\n", + "encoded_text = layers.LSTM(32)(embedded_text)\n", "\n", "question_input = Input(shape=(None,), dtype = 'int32', name = 'question')\n", - "embedded_question = layers.Embedding(32, question_vocabulary_size)(question_input)\n", - "encoded_question = layers.LSTM(16)(embedded_question)\n", + "embedded_question = layers.Embedding(question_vocabulary_size,32)(question_input)\n", + "encoded_question = layers.LSTM(16)(embedded_question)\n", "\n", "# concatenate two inputs and give output\n", "concatenated = layers.concatenate([encoded_text, encoded_question], axis=-1)\n", "answer = layers.Dense(answer_vocabulary_size, activation='softmax')(concatenated)\n", "\n", "model = Model([text_input, question_input], answer)\n", + "\n", "model.compile(optimizer = 'rmsprop', loss='categorical_crossentropy', metrics=['acc'])\n", "print(model.summary())" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Epoch 1/10\n" + "Epoch 1/10\n", + "8/8 [==============================] - 2s 37ms/step - loss: 1553.1508 - acc: 0.0012\n", + "Epoch 2/10\n", + "8/8 [==============================] - 0s 39ms/step - loss: 1556.4614 - acc: 0.0000e+00\n", + "Epoch 3/10\n", + "8/8 [==============================] - 0s 38ms/step - loss: 1562.3912 - acc: 0.0000e+00\n", + "Epoch 4/10\n", + "8/8 [==============================] - 0s 36ms/step - loss: 1568.0669 - acc: 0.0000e+00\n", + "Epoch 5/10\n", + "8/8 [==============================] - 0s 37ms/step - loss: 1570.9047 - acc: 0.0000e+00\n", + "Epoch 6/10\n", + "8/8 [==============================] - 0s 37ms/step - loss: 1569.9131 - acc: 0.0000e+00\n", + "Epoch 7/10\n", + "8/8 [==============================] - 0s 40ms/step - loss: 1575.4770 - acc: 0.0000e+00\n", + "Epoch 8/10\n", + "8/8 [==============================] - 0s 43ms/step - loss: 1572.6059 - acc: 0.0000e+00\n", + "Epoch 9/10\n", + "8/8 [==============================] - 0s 35ms/step - loss: 1572.1957 - acc: 0.0000e+00\n", + "Epoch 10/10\n", + "8/8 [==============================] - 0s 36ms/step - loss: 1572.1025 - acc: 0.0000e+00\n" ] }, { - "ename": "InvalidArgumentError", - "evalue": " indices[126,0] = 9504 is not in [0, 32)\n\t [[node model_8/embedding_24/embedding_lookup (defined at :13) ]] [Op:__inference_train_function_44332]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node model_8/embedding_24/embedding_lookup:\n model_8/embedding_24/embedding_lookup/42066 (defined at /Users/chaowu/opt/anaconda3/lib/python3.8/contextlib.py:113)\n\nFunction call stack:\ntrain_function\n", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mInvalidArgumentError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquestion\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0manswers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m128\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 14\u001b[0m \u001b[0;31m#model.fit({\"text\":text, \"question\":question}, answers, epochs=10, batch_size=128)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m 1098\u001b[0m _r=1):\n\u001b[1;32m 1099\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1100\u001b[0;31m \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1101\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1102\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 826\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 827\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 828\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 829\u001b[0m \u001b[0mcompiler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"xla\"\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_experimental_compile\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m\"nonXla\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 830\u001b[0m \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 853\u001b[0m \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 854\u001b[0m \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 855\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 856\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 857\u001b[0m \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 2940\u001b[0m (graph_function,\n\u001b[1;32m 2941\u001b[0m filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[0;32m-> 2942\u001b[0;31m return graph_function._call_flat(\n\u001b[0m\u001b[1;32m 2943\u001b[0m filtered_flat_args, captured_inputs=graph_function.captured_inputs) # pylint: disable=protected-access\n\u001b[1;32m 2944\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m 1916\u001b[0m and executing_eagerly):\n\u001b[1;32m 1917\u001b[0m \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1918\u001b[0;31m return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[1;32m 1919\u001b[0m ctx, args, cancellation_manager=cancellation_manager))\n\u001b[1;32m 1920\u001b[0m forward_backward = self._select_forward_and_backward_functions(\n", - "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m 553\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 554\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 555\u001b[0;31m outputs = execute.execute(\n\u001b[0m\u001b[1;32m 556\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 557\u001b[0m \u001b[0mnum_outputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[1;32m 60\u001b[0m inputs, attrs, num_outputs)\n\u001b[1;32m 61\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mInvalidArgumentError\u001b[0m: indices[126,0] = 9504 is not in [0, 32)\n\t [[node model_8/embedding_24/embedding_lookup (defined at :13) ]] [Op:__inference_train_function_44332]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node model_8/embedding_24/embedding_lookup:\n model_8/embedding_24/embedding_lookup/42066 (defined at /Users/chaowu/opt/anaconda3/lib/python3.8/contextlib.py:113)\n\nFunction call stack:\ntrain_function\n" - ] + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -113,7 +124,7 @@ "\n", "question = np.random.randint(1, question_vocabulary_size, size = (num_samples, max_length))\n", "\n", - "answers = np.random.randint(0,1, size = (num_samples, answer_vocabulary_size))\n", + "answers = np.random.randint(0,2, size = (num_samples, answer_vocabulary_size))\n", "\n", "\n", "model.fit([text, question], answers, epochs = 10, batch_size = 128)\n", @@ -122,9 +133,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [] }, { From f0e30cbd35ad4045acc3e7067f8969d802e0555e Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Sun, 30 May 2021 12:18:17 -0400 Subject: [PATCH 4/6] adding multi-output model --- 7.1.3-multi-output-model.ipynb | 157 +++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 7.1.3-multi-output-model.ipynb diff --git a/7.1.3-multi-output-model.ipynb b/7.1.3-multi-output-model.ipynb new file mode 100644 index 0000000000..59228e0b2d --- /dev/null +++ b/7.1.3-multi-output-model.ipynb @@ -0,0 +1,157 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# multiple-output models\n", + "## such as a network that takes as input a series of social media posts from a single anonymous person and tries to predict attributes of that person, such as age, gender, and income level " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from keras import layers\n", + "from keras import Input\n", + "from keras.models import Model" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "vocabulary_size=50000\n", + "num_income_groups=10\n", + "\n", + "post_input = Input(shape = (None,), dtype=\"int32\", name='posts')\n", + "embedded_posts = layers.Embedding(256, vocabulary_size)(post_input)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "x = layers.Conv1D(128, 5, activation='relu')(embedded_posts)\n", + "x = layers.MaxPooling1D(5)(x)\n", + "x = layers.Conv1D(256, 5, activation='relu')(x)\n", + "x = layers.Conv1D(256, 5, activation='relu')(x)\n", + "x = layers.MaxPooling1D(5)(x)\n", + "x = layers.Conv1D(256, 5, activation='relu')(x)\n", + "x = layers.Conv1D(256, 5, activation='relu')(x)\n", + "x = layers.GlobalMaxPooling1D()(x)\n", + "x = layers.Dense(128, activation='relu')(x)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "age_prediction = layers.Dense(1, name='age')(x)\n", + "income_prediction = layers.Dense(num_income_groups, activation ='softmax', name='income')(x)\n", + "gender_prediction = layers.Dense(1, activation ='sigmoid', name='gender')(x)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "model = Model(post_input,[age_prediction, income_prediction, gender_prediction])\n", + "model.compile(optimizer=\"rmsprop\",\n", + " loss = {'age':\"mse\",\n", + " \"income\":\"categorical_crossentropy\",\n", + " \"gender\":\"binary_crossentropy\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"model\"\n", + "__________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + "posts (InputLayer) [(None, None)] 0 \n", + "__________________________________________________________________________________________________\n", + "embedding (Embedding) (None, None, 50000) 12800000 posts[0][0] \n", + "__________________________________________________________________________________________________\n", + "conv1d (Conv1D) (None, None, 128) 32000128 embedding[0][0] \n", + "__________________________________________________________________________________________________\n", + "max_pooling1d (MaxPooling1D) (None, None, 128) 0 conv1d[0][0] \n", + "__________________________________________________________________________________________________\n", + "conv1d_1 (Conv1D) (None, None, 256) 164096 max_pooling1d[0][0] \n", + "__________________________________________________________________________________________________\n", + "conv1d_2 (Conv1D) (None, None, 256) 327936 conv1d_1[0][0] \n", + "__________________________________________________________________________________________________\n", + "max_pooling1d_1 (MaxPooling1D) (None, None, 256) 0 conv1d_2[0][0] \n", + "__________________________________________________________________________________________________\n", + "conv1d_3 (Conv1D) (None, None, 256) 327936 max_pooling1d_1[0][0] \n", + "__________________________________________________________________________________________________\n", + "conv1d_4 (Conv1D) (None, None, 256) 327936 conv1d_3[0][0] \n", + "__________________________________________________________________________________________________\n", + "global_max_pooling1d (GlobalMax (None, 256) 0 conv1d_4[0][0] \n", + "__________________________________________________________________________________________________\n", + "dense (Dense) (None, 128) 32896 global_max_pooling1d[0][0] \n", + "__________________________________________________________________________________________________\n", + "age (Dense) (None, 1) 129 dense[0][0] \n", + "__________________________________________________________________________________________________\n", + "income (Dense) (None, 10) 1290 dense[0][0] \n", + "__________________________________________________________________________________________________\n", + "gender (Dense) (None, 1) 129 dense[0][0] \n", + "==================================================================================================\n", + "Total params: 45,982,476\n", + "Trainable params: 45,982,476\n", + "Non-trainable params: 0\n", + "__________________________________________________________________________________________________\n" + ] + } + ], + "source": [ + "model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 325a592c57c59236435bd4d33da31dce201eb8ac Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Sun, 30 May 2021 12:22:57 -0400 Subject: [PATCH 5/6] adding loss_weights to balance the objective function --- 7.1.3-multi-output-model.ipynb | 36 ++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/7.1.3-multi-output-model.ipynb b/7.1.3-multi-output-model.ipynb index 59228e0b2d..28cf0c3d43 100644 --- a/7.1.3-multi-output-model.ipynb +++ b/7.1.3-multi-output-model.ipynb @@ -10,7 +10,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -21,7 +21,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -34,7 +34,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -51,7 +51,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -62,15 +62,33 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model = Model(post_input,[age_prediction, income_prediction, gender_prediction])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "#To remedy this, you can assign different levels of importance to the loss values in their contribution to the final loss. This is useful in particular if the losses’ values use different scales. For instance, the mean squared error (MSE) loss used for the age-regression task typically takes a value around 3–5, whereas the cross- entropy loss used for the gender-classification task can be as low as 0.1. In such a situa- tion, to balance the contribution of the different losses, you can assign a weight of 10 to the crossentropy loss and a weight of 0.25 to the MSE loss." + ] + }, + { + "cell_type": "code", + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ - "model = Model(post_input,[age_prediction, income_prediction, gender_prediction])\n", "model.compile(optimizer=\"rmsprop\",\n", " loss = {'age':\"mse\",\n", " \"income\":\"categorical_crossentropy\",\n", - " \"gender\":\"binary_crossentropy\"})" + " \"gender\":\"binary_crossentropy\"},\n", + " loss_weights={\"age\":0.25, \"income\":1, \"gender\":10})" ] }, { @@ -130,7 +148,9 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "#model.fit(posts, [age_targets, income_targets, gender_targets], epochs=10, batch_size=64)" + ] } ], "metadata": { From 016b27d172c2cda96f381bbecbb344202a11f680 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Mon, 31 May 2021 15:36:30 -0400 Subject: [PATCH 6/6] Update README.md Adding chapter 7 jupyter notebook. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 6039b0d710..0fdae5399b 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,10 @@ These notebooks use Python 3.6 and Keras 2.0.8. They were generated on a p2.xlar * [6.2: Understanding RNNs](http://nbviewer.jupyter.org/github/fchollet/deep-learning-with-python-notebooks/blob/master/6.2-understanding-recurrent-neural-networks.ipynb) * [6.3: Advanced usage of RNNs](http://nbviewer.jupyter.org/github/fchollet/deep-learning-with-python-notebooks/blob/master/6.3-advanced-usage-of-recurrent-neural-networks.ipynb) * [6.4: Sequence processing with convnets](http://nbviewer.jupyter.org/github/fchollet/deep-learning-with-python-notebooks/blob/master/6.4-sequence-processing-with-convnets.ipynb) +* Chapter 7: + * [7.1.1 Introduction to functional API] + * [7.1.2 Build a concatenate network] + * [7.1.3. Multi-output model] * Chapter 8: * [8.1: Text generation with LSTM](http://nbviewer.jupyter.org/github/fchollet/deep-learning-with-python-notebooks/blob/master/8.1-text-generation-with-lstm.ipynb) * [8.2: Deep dream](http://nbviewer.jupyter.org/github/fchollet/deep-learning-with-python-notebooks/blob/master/8.2-deep-dream.ipynb)