identify_silence.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.spatial.distance import cdist\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.externals import joblib\n",
    "\n",
    "from youtube_audioset import get_data, get_recursive_sound_names, get_all_sound_names\n",
    "from youtube_audioset import explosion_sounds, motor_sounds, wood_sounds, human_sounds, nature_sounds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ambient_sounds, impact_sounds = get_all_sound_names()\n",
    "\n",
    "explosion_sounds = get_recursive_sound_names(explosion_sounds)\n",
    "motor_sounds = get_recursive_sound_names(motor_sounds)\n",
    "wood_sounds = get_recursive_sound_names(wood_sounds)\n",
    "human_sounds = get_recursive_sound_names(human_sounds)\n",
    "nature_sounds = get_recursive_sound_names(nature_sounds)\n",
    "\n",
    "df, labels_binarized = get_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print df.shape[0], \"examples\"\n",
    "\n",
    "print \"Percentage Impact Sounds:\", (labels_binarized[impact_sounds].sum(axis=1) > 0).mean()\n",
    "print \"Percentage Ambient Sounds:\", (labels_binarized[ambient_sounds].sum(axis=1) > 0).mean()\n",
    "\n",
    "labels_binarized.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# df_filtered = df.loc[df.features.apply(lambda x: x.shape[0] == 10)]\n",
    "df_filtered = df.loc[df['labels'].apply(lambda x: (len(x) == 1)) & df.features.apply(lambda x: x.shape[0] == 10)]\n",
    "labels_filtered = labels_binarized.loc[df_filtered.index,:]\n",
    "\n",
    "X_total = np.array(df_filtered.features.apply(lambda x: x.flatten()).tolist())\n",
    "all_silence_ind = labels_filtered.Silence.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "pca_ = PCA(random_state=42)\n",
    "x_pca_ = pca_.fit_transform(X_total[all_silence_ind == 1,:].reshape((-1,128)))\n",
    "\n",
    "kmeans_ = KMeans(n_clusters=3, random_state=43)\n",
    "kgroup_ = kmeans_.fit_predict(X_total[all_silence_ind == 1,:].reshape((-1,128)))\n",
    "\n",
    "plt.clf()\n",
    "plt.figure(figsize=(20,20))\n",
    "\n",
    "plt.scatter(x_pca_[:,0], x_pca_[:,1],\n",
    "            c=kgroup_,\n",
    "            s=100, edgecolors='none')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's take the centroid of the first group and use that to identify an embedding for silence.  We don't look at the other groups because they may have sounds the we don't have labels for."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "silence_embedding = X_total[all_silence_ind == 1,:].reshape((-1,128))[kgroup_ == 0,:].mean(axis=0)\n",
    "\n",
    "silence_embedding.round()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll use the euclidean distance from this embedding to determine silence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "distance_from_silence = np.sqrt(np.square(X_total[all_silence_ind == 1,:].reshape((-1,128))[kgroup_ == 0,:] - \\\n",
    "                                  silence_embedding).sum(axis=1))\n",
    "\n",
    "distance_from_silence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.hist(distance_from_silence)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll need to zoom in on the histogram to find a good threshold."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.hist(distance_from_silence[distance_from_silence < 100], bins=20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "20 seems like a good distance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "silence_embedding_euclidean_distance_threshold = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cosine_distance_from_silence = cdist(X_total[all_silence_ind == 1,:].reshape((-1,128))[kgroup_ == 0,:],\n",
    "                                     silence_embedding.reshape(1,-1), 'cosine').reshape(-1)\n",
    "\n",
    "cosine_distance_from_silence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.hist(cosine_distance_from_silence[cosine_distance_from_silence < 0.02], bins = 20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "0.01 seems like a generous cosine distance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "silence_embedding_cosine_distance_threshold = 0.01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's save this embedding value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if not os.path.exists('parameter'):\n",
    "    os.makedirs('parameter')\n",
    "\n",
    "joblib.dump(silence_embedding, \"parameter/silence_embedding.pkl\")\n",
    "joblib.dump(silence_embedding_cosine_distance_threshold, \"parameter/silence_embedding_cosine_distance_threshold.pkl\")\n",
    "joblib.dump(silence_embedding_euclidean_distance_threshold, \"parameter/silence_embedding_euclidean_distance_threshold.pkl\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}