Amazon_Deep_Composer_32_Bars_Version.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "pycharm": {
      "stem_cell": {
        "cell_type": "raw",
        "metadata": {
          "collapsed": false
        },
        "source": []
      }
    },
    "colab": {
      "name": "Amazon-Deep-Composer.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/asigalov61/Amazon-Deep-Composer/blob/master/Amazon_Deep_Composer_32_Bars_Version.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rQKt8c_nv5RB",
        "colab_type": "text"
      },
      "source": [
        "#Amazon Deep Composer\n",
        "\n",
        "## Training a custom AR-CNN model \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fTagEO004Tvz",
        "colab_type": "text"
      },
      "source": [
        "#Start Here"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Mngh-Xnv5RC",
        "colab_type": "text"
      },
      "source": [
        "## Installing dependencies\n",
        "First, let's install and import all of the Python packages that we will use in this tutorial."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "pycharm": {
          "is_executing": false
        },
        "id": "51DsSASCv5RD",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Environment and Dependencies Setup. Please be patient as it may take a while. You may also need to run this section twice or trice if there are any errors. And in some cases, Runtime also needs to be restarted\n",
        "# The MIT-Zero License\n",
        "\n",
        "# Permission is hereby granted, free of charge, to any person obtaining a copy\n",
        "# of this software and associated documentation files (the \"Software\"), to deal\n",
        "# in the Software without restriction, including without limitation the rights\n",
        "# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n",
        "# copies of the Software, and to permit persons to whom the Software is\n",
        "# furnished to do so.\n",
        "\n",
        "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
        "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n",
        "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n",
        "# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n",
        "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n",
        "# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n",
        "# THE SOFTWARE.\n",
        "\n",
        "\n",
        "# Create the environment and install required packages\n",
        "!git clone https://github.com/asigalov61/aws-deepcomposer-samples.git\n",
        "%cd /content/aws-deepcomposer-samples/ar-cnn\n",
        "%tensorflow_version 1.x\n",
        "!pip install -r requirements.txt\n",
        "!pip install tensorflow-gpu==1.15.3\n",
        "!pip install pyFluidSynth\n",
        "!apt install fluidsynth\n",
        "!pip install midi2audio\n",
        "!pip install utils\n",
        "!cp /usr/share/sounds/sf2/FluidR3_GM.sf2 /content/font.sf2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "pycharm": {
          "is_executing": false,
          "name": "#%%\n"
        },
        "id": "oE1KlxnZv5RH",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Import all modules and other necessary code\n",
        "number_of_bars_to_augment = 32 #@param {type:\"slider\", min:1, max:128, step:1}\n",
        "\n",
        "# Imports\n",
        "\n",
        "from google.colab import output\n",
        "from IPython.display import display, Javascript, HTML, Audio\n",
        "import os\n",
        "import glob\n",
        "import json\n",
        "import numpy as np\n",
        "import keras\n",
        "from enum import Enum\n",
        "from keras.models import Model\n",
        "from keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D, concatenate, BatchNormalization, Dropout\n",
        "from keras.optimizers import Adam, RMSprop\n",
        "from keras import backend as K\n",
        "from random import randrange\n",
        "import random\n",
        "import math\n",
        "import pypianoroll\n",
        "from utils.midi_utils import play_midi, plot_pianoroll, get_music_metrics, process_pianoroll, process_midi\n",
        "\n",
        "from augmentation import AddAndRemoveAPercentageOfNotes\n",
        "from data_generator import PianoRollGenerator\n",
        "from utils.generate_training_plots import GenerateTrainingPlots\n",
        "#from inference import Inference\n",
        "from model import OptimizerType\n",
        "from model import ArCnnModel\n",
        "from midi2audio import FluidSynth\n",
        "\n",
        "class Constants():\n",
        "    # Make it a multiple of the batch size for best (balanced) performance\n",
        "    samples_per_ground_truth_data_item = 8\n",
        "    training_validation_split = 0.9\n",
        "    # Number of Bars\n",
        "    bars = number_of_bars_to_augment\n",
        "    # Number of Beats Per Bar\n",
        "    beats_per_bar = 4\n",
        "    beat_resolution = 4\n",
        "    # number of bars to be shifted\n",
        "    bars_shifted_per_sample = number_of_bars_to_augment\n",
        "    # Total number of pitches in a Pianoroll\n",
        "    number_of_pitches = 128\n",
        "    # Total number of Tracks\n",
        "    number_of_channels = 1\n",
        "    output_file_path = \"output_midi.mid\"\n",
        "    tempo = 100  # 100 bpm\n",
        "\n",
        "!mkdir /content/aws-deepcomposer-samples/ar-cnn/input     "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jmf4AI3Pv5RK",
        "colab_type": "text"
      },
      "source": [
        "### Download and Unzip Training DataSet\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y0_hvijdlhKE",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title (Best Choice/Works best stand-alone) Super Piano 2 Performance DataSet 320 MIDIs \n",
        "%cd /content/aws-deepcomposer-samples/ar-cnn/input\n",
        "!wget 'https://github.com/asigalov61/AlexMIDIDataSet/raw/master/Super-Piano-2-Performance-DataSet-CC-BY-NC-SA.zip'\n",
        "!unzip -j 'Super-Piano-2-Performance-DataSet-CC-BY-NC-SA.zip'\n",
        "!rm 'Super-Piano-2-Performance-DataSet-CC-BY-NC-SA.zip'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MolE6nN146om",
        "colab_type": "text"
      },
      "source": [
        "# Preprocess and Parse Training DataSet"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "pycharm": {
          "is_executing": false
        },
        "id": "Lzj8VkNLv5RN",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Import the MIDI files from the data_dir and save them with the midi_files variable  \n",
        "data_dir = '/content/aws-deepcomposer-samples/ar-cnn/input/*.*'\n",
        "midi_files = glob.glob(data_dir)\n",
        "\n",
        "#Finds our random MIDI file from the midi_files variable and then plays it\n",
        "#Note: To listen to multiple samples from the Bach dataset, you can run this cell over and over again. \n",
        "random_midi = randrange(len(midi_files))\n",
        "play_midi(midi_files[random_midi])\n",
        "#print(midi_files)\n",
        "print(midi_files[random_midi])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ckZvLi3Zv5RS",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Generate MIDI file samples and shuffle the DataSet\n",
        "def generate_samples(midi_files, bars, beats_per_bar, beat_resolution, bars_shifted_per_sample):\n",
        "    \"\"\"\n",
        "    dataset_files: All files in the dataset\n",
        "    return: piano roll samples sized to X bars\n",
        "    \"\"\"\n",
        "    timesteps_per_nbars = bars * beats_per_bar * beat_resolution\n",
        "    time_steps_shifted_per_sample = bars_shifted_per_sample * beats_per_bar * beat_resolution\n",
        "    samples = []\n",
        "    for midi_file in midi_files:\n",
        "        pianoroll = process_midi(midi_file, beat_resolution) # Parse the MIDI file and get the piano roll\n",
        "        samples.extend(process_pianoroll(pianoroll, time_steps_shifted_per_sample, timesteps_per_nbars))\n",
        "    return samples\n",
        "\n",
        "# Saving the generated samples into a dataset variable \n",
        "dataset_samples = generate_samples(midi_files, Constants.bars, Constants.beats_per_bar,Constants.beat_resolution, Constants.bars_shifted_per_sample)\n",
        "# Shuffle the dataset\n",
        "random.shuffle(dataset_samples);\n",
        "random_pianoroll = dataset_samples[randrange(len(dataset_samples))]\n",
        "\n",
        "plot_pianoroll(random_pianoroll, 4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YLHaOrByv5Rc",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Sampling from a uniform distribution\n",
        "sampling_lower_bound_remove = 0 \n",
        "sampling_upper_bound_remove = 100\n",
        "sampling_lower_bound_add = 1\n",
        "sampling_upper_bound_add = 1.5"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q2tdHvSxJlDb",
        "colab_type": "text"
      },
      "source": [
        "#Create, Setup the Model. Setup all (Hyper)Parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XAQY-2yLv5SZ",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Calculate Training and Validation sampling lengths\n",
        "dataset_size = len(dataset_samples)\n",
        "dataset_split = math.floor(dataset_size * Constants.training_validation_split) \n",
        "\n",
        "training_samples = dataset_samples[0:dataset_split]\n",
        "print(\"training samples length: {}\".format(len(training_samples)))\n",
        "validation_samples = dataset_samples[dataset_split + 1:dataset_size]\n",
        "print(\"validation samples length: {}\".format(len(validation_samples)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lQamt3I6v5Sd",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Specifying training hyperparameters. NOTE If you want to test that your model is training on your custom dataset, you can decrease the number of epochs down to 1 in this cell.\n",
        "batches_number = 24 #@param {type:\"slider\", min:1, max:128, step:1}\n",
        "number_of_training_epoch = 10 #@param {type:\"slider\", min:1, max:10, step:1}\n",
        "\n",
        "# Piano Roll Input Dimensions\n",
        "input_dim = (Constants.bars * Constants.beats_per_bar * Constants.beat_resolution, \n",
        "             Constants.number_of_pitches, \n",
        "             Constants.number_of_channels)\n",
        "# Number of Filters In The Convolution\n",
        "num_filters = 32\n",
        "# Growth Rate Of Number Of Filters At Each Convolution\n",
        "growth_factor = 2\n",
        "# Number Of Encoder And Decoder Layers\n",
        "num_layers = 5\n",
        "# A List Of Dropout Values At Each Encoder Layer\n",
        "dropout_rate_encoder = [0, 0.5, 0.5, 0.5, 0.5]\n",
        "# A List Of Dropout Values At Each Decoder Layer\n",
        "dropout_rate_decoder = [0.5, 0.5, 0.5, 0.5, 0]\n",
        "# A List Of Flags To Ensure If batch_normalization Should be performed At Each Encoder\n",
        "batch_norm_encoder = [True, True, True, True, False]\n",
        "# A List Of Flags To Ensure If batch_normalization Should be performed At Each Decoder\n",
        "batch_norm_decoder = [True, True, True, True, False]\n",
        "# Path to Pretrained Model If You Want To Initialize Weights Of The Network With The Pretrained Model\n",
        "pre_trained = False\n",
        "# Learning Rate Of The Model\n",
        "learning_rate = 0.001\n",
        "# Optimizer To Use While Training The Model\n",
        "optimizer_enum = OptimizerType.ADAM\n",
        "# Batch Size\n",
        "batch_size = batches_number\n",
        "# Number Of Epochs\n",
        "epochs = number_of_training_epoch"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q7qqyXuyv5Sg",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Calculate the number of Batch Iterations Before A Training Epoch Is Considered Finished\n",
        "steps_per_epoch = int(\n",
        "    len(training_samples) * Constants.samples_per_ground_truth_data_item / int(batch_size))\n",
        "\n",
        "print(\"The Total Number Of Steps Per Epoch Are: \"+ str(steps_per_epoch))\n",
        "\n",
        "# Total Number Of Time Steps\n",
        "n_timesteps = Constants.bars * Constants.beat_resolution * Constants.beats_per_bar"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g6zlQvcUv5Sj",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Creating the data generators that perform data augmentation. To create the input piano rolls during training, we need data generators for both the training and validation samples. For our purposes, we use a custom data generator to perform data augmentation.\n",
        "# Training Data Generator\n",
        "training_data_generator = PianoRollGenerator(sample_list=training_samples,\n",
        "                                             sampling_lower_bound_remove = sampling_lower_bound_remove,\n",
        "                                             sampling_upper_bound_remove = sampling_upper_bound_remove,\n",
        "                                             sampling_lower_bound_add = sampling_lower_bound_add,\n",
        "                                             sampling_upper_bound_add = sampling_upper_bound_add,\n",
        "                                             batch_size = batch_size,\n",
        "                                             bars = Constants.bars,\n",
        "                                             samples_per_data_item = Constants.samples_per_ground_truth_data_item,\n",
        "                                             beat_resolution = Constants.beat_resolution,\n",
        "                                             beats_per_bar = Constants.beats_per_bar,\n",
        "                                             number_of_pitches = Constants.number_of_pitches,\n",
        "                                             number_of_channels = Constants.number_of_channels)\n",
        "# Validation Data Generator\n",
        "validation_data_generator = PianoRollGenerator(sample_list = validation_samples,\n",
        "                                               sampling_lower_bound_remove = sampling_lower_bound_remove,\n",
        "                                               sampling_upper_bound_remove = sampling_upper_bound_remove,\n",
        "                                               sampling_lower_bound_add = sampling_lower_bound_add,\n",
        "                                               sampling_upper_bound_add = sampling_upper_bound_add,\n",
        "                                               batch_size = batch_size, \n",
        "                                               bars = Constants.bars,\n",
        "                                               samples_per_data_item = Constants.samples_per_ground_truth_data_item,\n",
        "                                               beat_resolution = Constants.beat_resolution,\n",
        "                                               beats_per_bar = Constants.beats_per_bar, \n",
        "                                               number_of_pitches = Constants.number_of_pitches,\n",
        "                                               number_of_channels = Constants.number_of_channels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R6Le2G_Tv5Sn",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Creating callbacks for the model and performing initial intitialization\n",
        "# Callback For Loss Plots \n",
        "plot_losses = GenerateTrainingPlots()\n",
        "## Checkpoint Path\n",
        "checkpoint_filepath =  '/content/aws-deepcomposer-samples/ar-cnn/trained-model-checkpoint.h5'\n",
        "\n",
        "# Callback For Saving Model Checkpoints \n",
        "model_checkpoint_callback = keras.callbacks.ModelCheckpoint(\n",
        "    filepath=checkpoint_filepath,\n",
        "    save_weights_only=False,\n",
        "    monitor='val_loss',\n",
        "    mode='min',\n",
        "    save_best_only=True)\n",
        "logdir = \"/content/aws-deepcomposer-samples/ar-cnn\"\n",
        "tensorboard_callback = keras.callbacks.TensorBoard(log_dir=logdir)\n",
        "# Create A List Of Callbacks\n",
        "callbacks_list = [plot_losses, model_checkpoint_callback, tensorboard_callback]\n",
        "\n",
        "\n",
        "# Create A Model Instance\n",
        "MusicModel = ArCnnModel(input_dim = input_dim,\n",
        "                        num_filters = num_filters,\n",
        "                        growth_factor = growth_factor,\n",
        "                        num_layers = num_layers,\n",
        "                        dropout_rate_encoder = dropout_rate_encoder,\n",
        "                        dropout_rate_decoder = dropout_rate_decoder,\n",
        "                        batch_norm_encoder = batch_norm_encoder,\n",
        "                        batch_norm_decoder = batch_norm_decoder,\n",
        "                        pre_trained = pre_trained,\n",
        "                        learning_rate = learning_rate,\n",
        "                        optimizer_enum = optimizer_enum)\n",
        "\n",
        "model = MusicModel.build_model()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ARj1GMDZp7Jq",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Tensorboard Graphs and Stats\n",
        "# Load the TensorBoard notebook extension\n",
        "%reload_ext tensorboard\n",
        "import tensorflow as tf\n",
        "import datetime, os\n",
        "%tensorboard --logdir '/content/aws-deepcomposer-samples/ar-cnn'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tq4RBl6uv5St",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Main Training Loop. In the following cell, you start training your model. NOTE: Training times can vary greatly based on the parameters that you have chosen and the notebook instance type that you chose when launching this notebook.\n",
        "# Start Training\n",
        "#logdir = \"/content/aws-deepcomposer-samples/ar-cnn\"\n",
        "tensorboard_callback = keras.callbacks.TensorBoard(log_dir=logdir)\n",
        "history = model.fit_generator(training_data_generator,\n",
        "                              use_multiprocessing = True,\n",
        "                              validation_data = validation_data_generator,\n",
        "                              steps_per_epoch = steps_per_epoch,\n",
        "                              epochs = epochs,\n",
        "                              callbacks = callbacks_list) #callbacks_list\n",
        "model.save('/content/aws-deepcomposer-samples/ar-cnn/trained-model-checkpoint.h5') "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0mVkk_8vTP7Z",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title (Optional) Save your model manually if necessary\n",
        "model.save('/content/aws-deepcomposer-samples/ar-cnn/trained-model-checkpoint.h5') "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aGMA84TTI8W8",
        "colab_type": "text"
      },
      "source": [
        "#Generate and Evaluate/Plot from the resulting model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JoJRiK-nv5Sv",
        "colab_type": "text"
      },
      "source": [
        "## Performing inference \n",
        "\n",
        "Congratulations! You have now trained your very own AR-CNN model to generate music. Now you can see how well your model will perform with an input melody. \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sa1a9b_xkyp-",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Inference Code Initialization Routine\n",
        "import os\n",
        "import logging\n",
        "import pypianoroll\n",
        "import keras\n",
        "import numpy as np\n",
        "from losses import Loss\n",
        "import copy\n",
        "\n",
        "logger = logging.getLogger(__name__)\n",
        "\n",
        "\n",
        "class Inference:\n",
        "    def __init__(self, model=None):\n",
        "        self.model = model\n",
        "        self.number_of_timesteps = (Constants.beat_resolution *\n",
        "                                    Constants.beats_per_bar * Constants.bars)\n",
        "\n",
        "    def load_model(self, model_path):\n",
        "        \"\"\"\n",
        "        Loads a trained keras model\n",
        "        Parameters\n",
        "        ----------\n",
        "        model_path : string\n",
        "            Full file path to the trained model\n",
        "        Returns\n",
        "        -------\n",
        "        None\n",
        "        \"\"\"\n",
        "        self.model = keras.models.load_model(model_path,\n",
        "                                             custom_objects={\n",
        "                                                 'built_in_softmax_kl_loss':\n",
        "                                                 Loss.built_in_softmax_kl_loss\n",
        "                                             },\n",
        "                                             compile=False)\n",
        "\n",
        "    @staticmethod\n",
        "    def convert_tensor_to_midi(tensor, tempo, output_file_path):\n",
        "        \"\"\"\n",
        "        Writes a pianoroll tensor to a midi file\n",
        "        Parameters\n",
        "        ----------\n",
        "        tensor : 2d numpy array\n",
        "            pianoroll to be converted to a midi\n",
        "        tempo : float\n",
        "            tempo to output\n",
        "        output_file_path : str\n",
        "            output midi file path\n",
        "        Returns\n",
        "        -------\n",
        "        None\n",
        "        \"\"\"\n",
        "\n",
        "        single_track = pypianoroll.Track(pianoroll=tensor)\n",
        "        multi_track = pypianoroll.Multitrack(\n",
        "            tracks=[single_track],\n",
        "            tempo=tempo,\n",
        "            beat_resolution=Constants.beat_resolution)\n",
        "        output_file_index = 0\n",
        "        while os.path.isfile(output_file_path.format(output_file_index)):\n",
        "            output_file_index += 1\n",
        "        multi_track.write(output_file_path.format(output_file_index))\n",
        "\n",
        "    @staticmethod\n",
        "    def get_indices(input_tensor, value):\n",
        "        \"\"\"\n",
        "        Parameters\n",
        "        ----------\n",
        "        input_tensor : 2d numpy array\n",
        "        value : int (either 1 or 0)\n",
        "        Returns\n",
        "        -------\n",
        "        indices_with_value : 2d array of indices in the input_tensor where the pixel value equals value (1 or 0).\n",
        "        \"\"\"\n",
        "        indices_with_value = np.argwhere(input_tensor.astype(np.bool_) == value)\n",
        "        return set(map(tuple, indices_with_value))\n",
        "\n",
        "    @staticmethod\n",
        "    def get_softmax(input_tensor, temperature):\n",
        "        \"\"\"\n",
        "        Gets the softmax of a tensor with temperature\n",
        "        Parameters\n",
        "        ----------\n",
        "        input_tensor : numpy array\n",
        "            original tensor (e.g. original predictions)\n",
        "        temperature : int\n",
        "            softmax temperature\n",
        "        Returns\n",
        "        -------\n",
        "        tensor : numpy array\n",
        "            softmax of input tensor with temperature\n",
        "        \"\"\"\n",
        "        tensor = input_tensor / temperature\n",
        "        tensor = np.exp(tensor)\n",
        "        tensor = tensor / np.sum(tensor)\n",
        "        return tensor\n",
        "\n",
        "    @staticmethod\n",
        "    def get_sampled_index(input_tensor):\n",
        "        \"\"\"\n",
        "        Gets a randomly chosen index from the input tensor\n",
        "        Parameters\n",
        "        ----------\n",
        "        input_tensor : numpy array\n",
        "            original tensor\n",
        "        Returns\n",
        "        -------\n",
        "        tensor : numpy array\n",
        "            softmax of input tensor with temperature\n",
        "        \"\"\"\n",
        "\n",
        "        sampled_index = np.random.choice(range(input_tensor.size),\n",
        "                                         1,\n",
        "                                         p=input_tensor.ravel())\n",
        "        sampled_index = np.unravel_index(sampled_index, input_tensor.shape)\n",
        "        return sampled_index\n",
        "\n",
        "    def generate_composition(self, input_midi_path, inference_params):\n",
        "        \"\"\"\n",
        "        Generates a new composition based on an old midi\n",
        "        Parameters\n",
        "        ----------\n",
        "        input_midi_path : str\n",
        "            input midi path\n",
        "        inference_params : json\n",
        "            JSON with inference parameters\n",
        "        Returns\n",
        "        -------\n",
        "        None\n",
        "        \"\"\"\n",
        "        try:\n",
        "            input_tensor = self.convert_midi_to_tensor(input_midi_path)\n",
        "            output_tensor = self.sample_multiple(\n",
        "                input_tensor, inference_params['temperature'],\n",
        "                inference_params['maxPercentageOfInitialNotesRemoved'],\n",
        "                inference_params['maxNotesAdded'],\n",
        "                inference_params['samplingIterations'])\n",
        "            self.convert_tensor_to_midi(output_tensor, Constants.tempo,\n",
        "                                        Constants.output_file_path)\n",
        "        except Exception:\n",
        "            logger.error(\"Unable to generate composition.\")\n",
        "            raise\n",
        "\n",
        "    def convert_midi_to_tensor(self, input_midi_path):\n",
        "        \"\"\"\n",
        "        Converts a midi to pianoroll tensor\n",
        "        Parameters\n",
        "        ----------\n",
        "        input_midi_path : string\n",
        "            Full file path to the input midi\n",
        "        Returns\n",
        "        -------\n",
        "        2d numpy array\n",
        "            2d tensor that is a pianoroll\n",
        "        \"\"\"\n",
        "\n",
        "        multi_track = pypianoroll.Multitrack(\n",
        "            beat_resolution=Constants.beat_resolution)\n",
        "        try:\n",
        "            multi_track.parse_midi(input_midi_path,\n",
        "                                   algorithm='custom',\n",
        "                                   first_beat_time=0)\n",
        "        except Exception as e:\n",
        "            logger.error(\"Failed to parse the MIDI file.\")\n",
        "\n",
        "        if len(multi_track.tracks) > 1:\n",
        "            logger.error(\"Input MIDI file has more than 1 track.\")\n",
        "\n",
        "        multi_track.pad_to_multiple(self.number_of_timesteps)\n",
        "        multi_track.binarize()\n",
        "        pianoroll = multi_track.tracks[0].pianoroll\n",
        "\n",
        "        if pianoroll.shape[0] > self.number_of_timesteps:\n",
        "            logger.error(\"Input MIDI file is longer than 8 bars.\")\n",
        "\n",
        "        # truncate\n",
        "        tensor = pianoroll[0:self.number_of_timesteps, ]\n",
        "        tensor = np.expand_dims(tensor, axis=0)\n",
        "        tensor = np.expand_dims(tensor, axis=3)\n",
        "\n",
        "        return tensor\n",
        "\n",
        "    def mask_not_allowed_notes(self, current_input_indices, output_tensor):\n",
        "        \"\"\"\n",
        "        Masks notes in output tensor that cannot be added or removed\n",
        "        Parameters\n",
        "        ----------\n",
        "        current_input_indices : 2d numpy array\n",
        "          indices to be masked based on the current input that was fed to model\n",
        "        output_tensor : 2d numpy array\n",
        "          consists of probabilities that are predicted by the model\n",
        "        Returns\n",
        "        -------\n",
        "        2d numpy array - output tensor with not allowed notes masked\n",
        "        \"\"\"\n",
        "\n",
        "        if len(current_input_indices) != 0:\n",
        "            output_tensor[tuple(np.asarray(list(current_input_indices)).T)] = 0\n",
        "            if np.count_nonzero(output_tensor) != 0:\n",
        "                output_tensor = output_tensor / np.sum(output_tensor)\n",
        "        return output_tensor\n",
        "\n",
        "    def sample_multiple(self, input_tensor, temperature,\n",
        "                        max_removal_percentage, max_notes_to_add,\n",
        "                        number_of_iterations):\n",
        "        \"\"\"\n",
        "        Samples multiple times from an tensor.\n",
        "        Returns the final output tensor after X number of iterations.\n",
        "        Parameters\n",
        "        ----------\n",
        "        input_tensor : 2d numpy array\n",
        "            original tensor (i.e. user input melody)\n",
        "        temperature : float\n",
        "            temperature to apply before softmax during inference\n",
        "        max_removal_percentage : float\n",
        "            maximum percentage of notes that can be removed from the original input\n",
        "        max_notes_to_add : int\n",
        "            maximum number of notes that can be added to the original input\n",
        "        number_of_iterations : int\n",
        "            number of iterations to sample from the model predictions\n",
        "        Returns\n",
        "        -------\n",
        "        2d numpy array\n",
        "            output tensor (i.e. new composition)\n",
        "        \"\"\"\n",
        "\n",
        "        max_original_notes_to_remove = int(\n",
        "            max_removal_percentage * np.count_nonzero(input_tensor) / 100)\n",
        "        notes_removed_count = 0\n",
        "        notes_added_count = 0\n",
        "\n",
        "        original_input_one_indices = self.get_indices(input_tensor, 1)\n",
        "        original_input_zero_indices = self.get_indices(input_tensor, 0)\n",
        "\n",
        "        current_input_one_indices = copy.deepcopy(original_input_one_indices)\n",
        "        current_input_zero_indices = copy.deepcopy(original_input_zero_indices)\n",
        "\n",
        "        for _ in range(number_of_iterations):\n",
        "            input_tensor, notes_removed_count, notes_added_count = self.sample_notes_from_model(\n",
        "                input_tensor, max_original_notes_to_remove, max_notes_to_add,\n",
        "                temperature, notes_removed_count, notes_added_count,\n",
        "                original_input_one_indices, original_input_zero_indices,\n",
        "                current_input_zero_indices, current_input_one_indices)\n",
        "\n",
        "        return input_tensor.reshape(self.number_of_timesteps,\n",
        "                                    Constants.number_of_pitches)\n",
        "\n",
        "    def sample_notes_from_model(self,\n",
        "                                input_tensor,\n",
        "                                max_original_notes_to_remove,\n",
        "                                max_notes_to_add,\n",
        "                                temperature,\n",
        "                                notes_removed_count,\n",
        "                                notes_added_count,\n",
        "                                original_input_one_indices,\n",
        "                                original_input_zero_indices,\n",
        "                                current_input_zero_indices,\n",
        "                                current_input_one_indices,\n",
        "                                num_notes=1):\n",
        "        \"\"\"\n",
        "        Generates a sample from the tensor and return a new tensor\n",
        "        Modifies current_input_zero_indices, current_input_one_indices, and input_tensor\n",
        "        Parameters\n",
        "        ----------\n",
        "        input_tensor : 2d numpy array\n",
        "            input tensor to feed into the model\n",
        "        max_original_notes_to_remove : int\n",
        "            maximum number of notes to remove from the original input\n",
        "        max_notes_to_add : int\n",
        "            maximum number of notes that can be added to the original input\n",
        "        temperature : float\n",
        "            temperature to apply before softmax during inference\n",
        "        notes_removed_count : int\n",
        "            number of original notes that have been removed from input\n",
        "        notes_added_count : int\n",
        "            number of new notes that have been added to the input\n",
        "        original_input_one_indices : set of tuples\n",
        "            indices which have value 1 in original input\n",
        "        original_input_zero_indices : set of tuples\n",
        "            indices which have value 0 in original input\n",
        "        current_input_zero_indices : set of tuples\n",
        "            indices which have value 0 and were not part of the original input\n",
        "        current_input_one_indices : set of tuples\n",
        "            indices which have value 1 and were part of the original input\n",
        "        Returns\n",
        "        -------\n",
        "        input_tensor : 2d numpy array\n",
        "            output after samping from the model prediction\n",
        "        notes_removed_count : int\n",
        "            updated number of original notes removed\n",
        "        notes_added_count : int\n",
        "            updated number of new notes added\n",
        "        \"\"\"\n",
        "\n",
        "        output_tensor = self.model.predict([input_tensor])\n",
        "\n",
        "        # Apply temperature and softmax\n",
        "        output_tensor = self.get_softmax(output_tensor, temperature)\n",
        "\n",
        "        if notes_removed_count >= max_original_notes_to_remove:\n",
        "            # Mask all pixels that both have a note and were once part of the original input\n",
        "            output_tensor = self.mask_not_allowed_notes(current_input_one_indices, output_tensor)\n",
        "\n",
        "        if notes_added_count > max_notes_to_add:\n",
        "            # Mask all pixels that both do not have a note and were not once part of the original input\n",
        "            output_tensor = self.mask_not_allowed_notes(current_input_zero_indices, output_tensor)\n",
        "\n",
        "        if np.count_nonzero(output_tensor) == 0:\n",
        "            return input_tensor, notes_removed_count, notes_added_count\n",
        "\n",
        "        sampled_index = self.get_sampled_index(output_tensor)\n",
        "        sampled_index_transpose = tuple(np.array(sampled_index).T[0])\n",
        "\n",
        "        if input_tensor[sampled_index]:\n",
        "            # Check if the note being removed is from the original input\n",
        "            if notes_removed_count < max_original_notes_to_remove and (\n",
        "                sampled_index_transpose in original_input_one_indices):\n",
        "                notes_removed_count += 1\n",
        "                current_input_one_indices.remove(sampled_index_transpose)\n",
        "            elif tuple(sampled_index_transpose) not in original_input_one_indices:\n",
        "                notes_added_count -= 1\n",
        "                current_input_zero_indices.add(sampled_index_transpose)\n",
        "            input_tensor[sampled_index] = 0\n",
        "        else:\n",
        "            # Check if the note being added is not in original input\n",
        "            if sampled_index_transpose not in original_input_one_indices:\n",
        "                notes_added_count += 1\n",
        "                current_input_zero_indices.remove(sampled_index_transpose)\n",
        "            else:\n",
        "                notes_removed_count -= 1\n",
        "                current_input_one_indices.add(sampled_index_transpose)\n",
        "            input_tensor[sampled_index] = 1\n",
        "        input_tensor = input_tensor.astype(np.bool_)\n",
        "        return input_tensor, notes_removed_count, notes_added_count\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kXXl1SE4XjOt",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Loading a saved checkpoint file. To use your trained model, you will need to update the PATH variable in the cell below.\n",
        "full_path_to_checkpoint_file = \"/content/aws-deepcomposer-samples/ar-cnn/trained-model-checkpoint.h5\" #@param {type:\"string\"}\n",
        "\n",
        "%cd /content/aws-deepcomposer-samples/ar-cnn\n",
        "# Create An Inference Object\n",
        "inference_obj = Inference()\n",
        "# Load The Checkpoint\n",
        "checkpoint_var = full_path_to_checkpoint_file\n",
        "inference_obj.load_model(checkpoint_var)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vuqvqto6v5Sv",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title MODEL INFERENCE/GENERATION CONTROLS { run: \"auto\" }\n",
        "model_temperature = 2 #@param {type:\"slider\", min:0.1, max:6, step:0.1}\n",
        "percentage_of_max_removed_notes = 0 #@param {type:\"slider\", min:0, max:100, step:1}\n",
        "percentage_of_max_added_notes = 100 #@param {type:\"slider\", min:0, max:100, step:1}\n",
        "output_refinement_iterations = 100 #@param {type:\"slider\", min:0, max:100, step:1}\n",
        "#with open('inference_parameters.json') as json_file:\n",
        "#    inference_params = json.load(json_file)\n",
        "#print(inference_params)\n",
        "\n",
        "import json\n",
        "\n",
        "data = {\n",
        "  u\"temperature\": float(model_temperature),\n",
        "  u\"maxNotesAdded\": int(percentage_of_max_added_notes),\n",
        "  u\"maxPercentageOfInitialNotesRemoved\": int(percentage_of_max_removed_notes),\n",
        "  u\"samplingIterations\": int(output_refinement_iterations)\n",
        "}\n",
        "\n",
        "data = json.dumps(data) # dict to string\n",
        "inference_params = json.loads(data) # string to json\n",
        "#print (inference_params)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3nt_l2Ldv5S3",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Main Generation Loop\n",
        "desired_midi_file_full_path = \"/content/aws-deepcomposer-samples/ar-cnn/seed7.mid\" #@param {type:\"string\"}\n",
        "# Generate The Composition\n",
        "%cd /content/aws-deepcomposer-samples/ar-cnn\n",
        "!rm output_wav.wav output_midi.mid\n",
        "input_melody = desired_midi_file_full_path\n",
        "inference_obj.generate_composition(desired_midi_file_full_path, inference_params)\n",
        "FluidSynth(\"/content/font.sf2\").midi_to_audio('output_midi.mid','output_wav.wav')\n",
        "# set the src and play\n",
        "from google.colab import files\n",
        "files.download('/content/aws-deepcomposer-samples/ar-cnn/output_midi.mid')\n",
        "Audio(\"output_wav.wav\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eYGtPUMB1FxI",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Plot and Graph the Output :) Only first batch MIDI file is plotted and displayed \n",
        "graphs_length_inches = 18 #@param {type:\"slider\", min:0, max:20, step:1}\n",
        "notes_graph_height = 6 #@param {type:\"slider\", min:0, max:20, step:1}\n",
        "highest_displayed_pitch = 100 #@param {type:\"slider\", min:1, max:128, step:1}\n",
        "lowest_displayed_pitch = 10 #@param {type:\"slider\", min:1, max:128, step:1}\n",
        "\n",
        "import librosa\n",
        "import numpy as np\n",
        "import pretty_midi\n",
        "import pypianoroll\n",
        "from pypianoroll import Multitrack, Track\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "matplotlib.use('SVG')\n",
        "# For plotting\n",
        "import mir_eval.display\n",
        "import librosa.display\n",
        "%matplotlib inline\n",
        "output_melody = '/content/aws-deepcomposer-samples/ar-cnn/output_midi.mid'\n",
        "\n",
        "midi_data = pretty_midi.PrettyMIDI('/content/aws-deepcomposer-samples/ar-cnn/output_midi.mid')\n",
        "\n",
        "def plot_piano_roll(pm, start_pitch, end_pitch, fs=100):\n",
        "    # Use librosa's specshow function for displaying the piano roll\n",
        "    librosa.display.specshow(pm.get_piano_roll(fs)[start_pitch:end_pitch],\n",
        "                             hop_length=1, sr=fs, x_axis='time', y_axis='cqt_note',\n",
        "                             fmin=pretty_midi.note_number_to_hz(start_pitch))\n",
        "\n",
        "\n",
        "\n",
        "roll = np.zeros([int(graphs_length_inches), 128])\n",
        "# Plot the output\n",
        "\n",
        "track = Multitrack('/content/aws-deepcomposer-samples/ar-cnn/output_midi.mid', name='track')\n",
        "plt.figure(figsize=[graphs_length_inches, notes_graph_height])\n",
        "fig, ax = track.plot()\n",
        "fig.set_size_inches(graphs_length_inches, notes_graph_height)\n",
        "plt.figure(figsize=[graphs_length_inches, notes_graph_height])\n",
        "ax2 = plot_piano_roll(midi_data, lowest_displayed_pitch, highest_displayed_pitch)\n",
        "plt.show(block=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7GJJ1-B6v5S-",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Additional Input/Output Comparison Metrics of the specified MIDI file\n",
        "# Input Midi Metrics:\n",
        "print(\"The input midi metrics are:\")\n",
        "get_music_metrics(input_melody, beat_resolution=4)\n",
        "\n",
        "print(\"\\n\")\n",
        "# Generated Output Midi Metrics:\n",
        "print(\"The generated output midi metrics are:\")\n",
        "get_music_metrics(output_melody, beat_resolution=4)\n",
        "\n",
        "# Convert The Input and Generated Midi To Tensors (a matrix)\n",
        "input_pianoroll = process_midi(input_melody, beat_resolution=4)\n",
        "output_pianoroll = process_midi(output_melody, beat_resolution=4)\n",
        "\n",
        "# Plot Input Piano Roll\n",
        "plot_pianoroll(input_pianoroll, beat_resolution=4)\n",
        "\n",
        "# Plot Output Piano Roll\n",
        "plot_pianoroll(output_pianoroll, beat_resolution=4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BXspRCGnANpl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}