preprocess-4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IoT Project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Client</h3>\n",
       "<ul>\n",
       "  <li><b>Scheduler: </b>tcp://127.0.0.1:8786\n",
       "  <li><b>Dashboard: </b><a href='http://127.0.0.1:8787' target='_blank'>http://127.0.0.1:8787</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Cluster</h3>\n",
       "<ul>\n",
       "  <li><b>Workers: </b>4</li>\n",
       "  <li><b>Cores: </b>16</li>\n",
       "  <li><b>Memory: </b>68.72 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: scheduler='tcp://192.168.2.46:8786' processes=4 cores=16>"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "from scapy.all import *\n",
    "from IPy import IP as IPy\n",
    "import pprint\n",
    "from sklearn import preprocessing\n",
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from dask.distributed import Client, progress\n",
    "import dask.dataframe as dd\n",
    "#import dask.bag as db\n",
    "client = Client('127.0.0.1:8786')\n",
    "client"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Internet IP address <-> Organization mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "ASN_df = pd.read_csv(\"./ip2asn-combined.tsv\", sep='\\t', header=None)\n",
    "ASN_df.columns = ['start','end','asn','country','organization']\n",
    "\n",
    "# remove all the \"Not routed\" rows\n",
    "ASN_df = ASN_df[ASN_df['organization'] != 'Not routed']\n",
    "\n",
    "# add numerical representation for the start and end IP range for faster org search later\n",
    "ASN_df['start.dec'] = ASN_df['start'].apply(lambda x: float(IPy(x).strDec()))\n",
    "ASN_df['end.dec'] = ASN_df['end'].apply(lambda x: float(IPy(x).strDec()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Device labeling  \n",
    "The packet traces we are using come from my home network. I know what these devices are. I am going to import the device information for these devices and merge it with the network traffic dataframe we are using."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "OUI_df = pd.read_csv('oui.csv')\n",
    "known_devices = pd.read_csv('sydney_devices.csv')\n",
    "known_devices['OUI'] = known_devices['MAC address'].apply(lambda x: ''.join(x.upper().split(':')[0:3]))\n",
    "known_devices['Manufacturer Device Type'] = ([' '.join(row) for row in \n",
    "                        zip(known_devices[\"Manufacturer\"],known_devices[\"Device Type\"])])\n",
    "known_devices['MAC address'] = known_devices['MAC address'].apply(lambda x: x.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# baseline \n",
    "tshark_cmd = \"-T fields -E header=y -e frame.number -e frame.time -e frame.len -e frame.protocols\"\n",
    "\n",
    "# ethernet layer\n",
    "#tshark_cmd += \" -e eth.src_resolved -e eth.dst_resolved\"\n",
    "tshark_cmd += \" -e eth.src -e eth.dst -e eth.dst_resolved\"\n",
    "\n",
    "#add IP/TCP/UDP/ICMP layers\n",
    "tshark_cmd += \" -e ip.src -e ip.dst -e tcp.srcport -e tcp.dstport -e udp.srcport -e udp.dstport\"\n",
    "\n",
    "# add DSN / mDNS layer\n",
    "tshark_cmd += \" -e dns.qry.name -e dns.resp.name -e dns.cname -e dns.a\"\n",
    "\n",
    "# add HTTP layer\n",
    "tshark_cmd += \" -e http.request.method -e http.request.uri -e http.user_agent -e http.host\"\n",
    "\n",
    "# add SSL certificate layer\n",
    "tshark_cmd += \" -e x509sat.printableString -e x509sat.uTF8String\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!tshark -tud -N m -r ./packet_capture.pcap {tshark_cmd} > daghan.csv\n",
    "#!tshark -tud -N m -r ./IoT_Trafﬁc_UNSW_Sydney/train_large_2.pcap {tshark_cmd} > packets_train_large_2.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def determineZone(ip):\n",
    "    \"\"\"\n",
    "    This function determines if the IP \n",
    "    address is internal or public \n",
    "    according to RFC1918\n",
    "    \"\"\"    \n",
    "    if pd.notna(ip):\n",
    "        return IPy(ip.split(',')[0]).iptype()\n",
    "    else:\n",
    "        return ip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "def protoLang(line):\n",
    "    proto = line['protocol'] \n",
    "    size = \"size: \" + str(line['frame.len'])\n",
    "    message = \"\"\n",
    "    \n",
    "    try:\n",
    "        \n",
    "        ########################################\n",
    "        ### Extract base line protocol       ###\n",
    "        ########################################\n",
    "        \n",
    "        #UDP based?\n",
    "        if line['frame.protocols'].find('udp') > 0:     \n",
    "            src =  \" \".join(['device:',line['org.src'], 'identifier:', str(line['tail.src'])]) if line['zone.src'] == 'PRIVATE' \\\n",
    "                else \" \".join(['service:',line['org.src']])\n",
    "            srcport = 'port: ' + str(int(line['udp.srcport']))\n",
    "            dst =  \"\" if pd.isna(line['org.dst']) \\\n",
    "                    else (\" \".join(['device:', line['org.dst'],'identifier:', str(line['tail.dst'])]) if line['zone.dst'] == 'PRIVATE' \\\n",
    "                            else (\" \".join(['service:',line['org.src']]) if line['zone.dst'] == 'PUBLIC' else ''))\n",
    "            dstport = 'port: ' + str(int(line['udp.dstport']))\n",
    "        \n",
    "        #UDP based?\n",
    "        if line['frame.protocols'].find('tcp') > 0:     \n",
    "            src =  \" \".join(['device:',line['org.src'], 'identifier:', str(line['tail.src'])]) if line['zone.src'] == 'PRIVATE' \\\n",
    "                else \" \".join(['service:',line['org.src']])\n",
    "            srcport = 'port: ' + str(int(line['tcp.srcport']))\n",
    "            dst =  \"\" if pd.isna(line['org.dst']) \\\n",
    "                    else (\" \".join(['device:', line['org.dst'],'identifier:', str(line['tail.dst'])]) if line['zone.dst'] == 'PRIVATE' \\\n",
    "                            else (\" \".join(['service:',line['org.src']]) if line['zone.dst'] == 'PUBLIC' else ''))\n",
    "            dstport = 'port: ' + str(int(line['tcp.dstport']))\n",
    "        \n",
    "        ## ICMPv6\n",
    "        ## TODO: There is more information to be extracted here\n",
    "        elif proto == 'icmpv6':\n",
    "            src =  \" \".join([line['org.src'],line['tail.src']]) \n",
    "            return \" \".join(['device',src,proto,size]).lower()\n",
    "        \n",
    "        \n",
    "        #IP (L3) that is neither of those\n",
    "        elif line['frame.protocols'].find('ip') > 0:\n",
    "            src =  \" \".join(['device', line['org.src'],line['tail.src']]) if line['zone.src'] == 'PRIVATE' \\\n",
    "                    else \" \".join(['service', line['org.src']])\n",
    "            dst =  \"\" if pd.isna(line['org.dst']) \\\n",
    "                    else (\" \".join(['device', line['org.dst'],line['tail.dst']]) if line['zone.dst'] == 'PRIVATE' \\\n",
    "                            else (\" \".join(['service', line['org.src']]) if line['zone.dst'] == 'PUBLIC' else ''))\n",
    "        \n",
    "        #L2\n",
    "        elif line['frame.protocols'].find('ethertype') > 0:\n",
    "            src =  \" \".join(['device',line['org.src'],line['tail.src']]) \n",
    "            dst =  \" \".join(['device', line['org.dst'],line['tail.dst']]) if pd.notna(line['org.dst']) else ''\n",
    "        \n",
    "        #this should very rarely happen, if at all!\n",
    "        else:\n",
    "            print(\"undetected protocol(1): {}\".format(line))\n",
    "            return line['frame.protocols'].lower()\n",
    "        \n",
    "        \n",
    "        \n",
    "        ########################################\n",
    "        ### Extract higher layer protocol    ###\n",
    "        ### and additional meta data         ###\n",
    "        ########################################        \n",
    "         # x509ce (certificate exchange)\n",
    "        if line['frame.protocols'].find('x509') > 0:\n",
    "            proto = \"protocol: \" + proto\n",
    "            x509ce_message = ''\n",
    "            if pd.notna(line['x509sat.printableString']):\n",
    "                x509ce_message += line['x509sat.printableString']\n",
    "            if pd.notna(line['x509sat.uTF8String']):\n",
    "                x509ce_message += line['x509sat.uTF8String']\n",
    "            return \" \".join([src,srcport,proto,size,dst,dstport,x509ce_message]).lower()\n",
    "        \n",
    "        ## ssl \n",
    "        elif proto == 'ssl':\n",
    "            proto = \"protocol: \" + proto\n",
    "            return \" \".join([src,srcport,proto,size,dst,dstport]).lower()\n",
    "                \n",
    "        \n",
    "        ## tcp:data\n",
    "        elif line['frame.protocols'].find('tcp:data') > 0:\n",
    "            proto = \"protocol: tcp:data\"\n",
    "            return \" \".join([src,srcport,proto,size,dst,dstport]).lower()\n",
    "        \n",
    "        ## tcp:stun\n",
    "        ## TODO: stun.att.software \"stun.att.realm\": \"belkin.org\",\n",
    "        elif line['frame.protocols'].find('tcp:stun') > 0:\n",
    "            proto = 'protocol: tcp:stun'\n",
    "            return \" \".join([src,srcport,proto,size,dst,dstport]).lower()\n",
    "        \n",
    "        ## upd:data\n",
    "        ## TODO: Detect broadcasts\n",
    "        elif line['frame.protocols'].find('udp:data') > 0:\n",
    "            proto = 'protocol: udp:data'\n",
    "            return \" \".join([src,srcport,proto,size,dst,dstport]).lower()\n",
    "        \n",
    "        ## udp:nbns (netbios)\n",
    "        ## TODO: nbns.name\": \"MACBOOKAIR-7040<00> (Workstation\\/Redirector)\",\n",
    "        elif line['frame.protocols'].find('udp:nbns') > 0:\n",
    "            proto = 'protocol: udp:nbns'\n",
    "            return \" \".join([src,srcport,proto,size]).lower()\n",
    "        \n",
    "        ## upd:bootp (dhcp)\n",
    "        ## TODO: \"bootp.option.hostname\": \"amazon-c4475da2a\"\n",
    "        ## TODO: \"bootp.type\": \"2\" (1 is request, 2 is reply)\n",
    "        ## if it is a reply, add the dst IP address too\n",
    "        elif line['frame.protocols'].find('udp:bootp') > 0:\n",
    "            proto = 'protocol: udp:bootp'\n",
    "            return \" \".join([src,srcport,proto,size]).lower()\n",
    "        \n",
    "        ## udP:gquic\n",
    "        ## TODO: \"gquic.tag.sni\": \"0.docs.google.com\"\n",
    "        ## TODO: \"gquic.tag.uaid\": \"Chrome\\/65.0.3325.181 Intel Mac OS X 10_13_3\" \n",
    "        ## TODO: \"gquic.tag\": \"CHLO\" (client hello)\n",
    "        elif line['frame.protocols'].find('udp:quic') > 0:\n",
    "            proto = 'protocol: udp:gquic'\n",
    "            return \" \".join([src,srcport,proto,size,dst,dstport]).lower()\n",
    "        \n",
    "        ## ssdp (simple service discovery protocol)\n",
    "        ## TODO: http.server: \"Linux UPnP\\/1.0 Sonos\\/41.3-50131 (ZPS12)\n",
    "        ## TODO: http.unknown_header: \"HOUSEHOLD.SMARTSPEAKER.AUDIO: Sonos_hOcMvZ0JBvDVZz7BXZc5ILQAT5.Cd7MOjIUy3HWHWEXItIZ\\\\r\\\\n\",\n",
    "        ## TODO: http.request.full_uri: \"http:\\/\\/239.255.255.250:1900*\",\n",
    "        elif proto == 'ssdp':\n",
    "            proto = \"protocol: \" + proto\n",
    "            return \" \".join([src,srcport,proto,size,dst,dstport]).lower()\n",
    "        \n",
    "        ## db-lsp-disc:json (Dropbox Lan sync Discovery Protocol)\n",
    "        ## TODO: Detect and use Broadcast\n",
    "        ## TODO: eth.addr_resolved\": \"Broadcast\"\n",
    "        elif line['frame.protocols'].find('db-lsp-disc:json') > 0:\n",
    "            proto = 'protocol: db-lsp-disc'\n",
    "            return \" \".join([src,srcport,proto,size,dstport]).lower()\n",
    "        \n",
    "        ## ntp (network time protocol)\n",
    "        elif proto == 'ntp':\n",
    "            proto = \"protocol: \" + proto\n",
    "            return \" \".join([src,srcport,proto,size,dst,dstport]).lower()\n",
    "        \n",
    "        ## STP (spanning tree protocol)\n",
    "        elif proto == 'stp':\n",
    "            proto = \"protocol: \" + proto\n",
    "            return \" \".join([src,proto,size]).lower()\n",
    "        \n",
    "        ## mdns\n",
    "        elif proto == 'mdns':\n",
    "            proto = \"protocol: \" + proto\n",
    "            mdns_string =   (line['dns.qry.name'] + \" \" if pd.notna(line['dns.qry.name']) else '') + \\\n",
    "                            (line['dns.resp.name'] + \" \" if pd.notna(line['dns.resp.name']) else '') +\\\n",
    "                            (line['dns.cname'] + \" \" if pd.notna(line['dns.cname']) else '') + \\\n",
    "                            (line['dns.a'] if pd.notna(line['dns.a']) else '')    \n",
    "            return \" \".join([src,srcport,proto,size,mdns_string]).lower()\n",
    "        \n",
    "        ## igmp\n",
    "        ##TODO: look into IGMP\n",
    "        elif proto == 'igmp':\n",
    "            proto = \"protocol: \" + proto\n",
    "            return \" \".join([src,proto,dst]).lower()\n",
    "        \n",
    "        ## dns\n",
    "        elif proto == 'dns':\n",
    "            proto = \"protocol: \" + proto\n",
    "            dns_string =   (line['dns.qry.name'] + \" \" if pd.notna(line['dns.qry.name']) else '') + \\\n",
    "                            (line['dns.resp.name'] + \" \" if pd.notna(line['dns.resp.name']) else '') +\\\n",
    "                            (line['dns.cname'] + \" \" if pd.notna(line['dns.cname']) else '') + \\\n",
    "                            (line['dns.a'] if pd.notna(line['dns.a']) else '')\n",
    "            return \" \".join([src,srcport,proto,size,dst,dstport,dns_string]).lower()\n",
    "\n",
    "        ## http \n",
    "        elif proto == 'http':\n",
    "            proto = \"protocol: \" + proto\n",
    "            payload = (line['http.request.method'] + \" \" if pd.notna(line['http.request.method']) else '') + \\\n",
    "                    (line['http.request.uri'] + \" \" if pd.notna(line['http.request.uri']) else '') + \\\n",
    "                    (line['http.user_agent'] + \" \" if pd.notna(line['http.user_agent']) else '') + \\\n",
    "                    (line['http.host'] if pd.notna(line['http.host']) else '')\n",
    "            return \" \".join([src,srcport,proto,size,dst,dstport,payload]).lower()\n",
    "        \n",
    "        ## http:data\n",
    "        ## TODO: add http.file_data content\n",
    "        elif line['frame.protocols'].find('http:data') > 0:\n",
    "            proto = 'protocol: http:data'\n",
    "            payload = (line['http.request.method'] + \" \" if pd.notna(line['http.request.method']) else '') + \\\n",
    "                    (line['http.request.uri'] + \" \" if pd.notna(line['http.request.uri']) else '') + \\\n",
    "                    (line['http.user_agent'] + \" \" if pd.notna(line['http.user_agent']) else '') + \\\n",
    "                    (line['http.host'] if pd.notna(line['http.host']) else '')\n",
    "            return \" \".join([src,srcport,proto,size,dst,dstport,payload]).lower()\n",
    "        \n",
    "        ## http:media\n",
    "        ## TODO: add  http.content_type , http.content_length\n",
    "        elif line['frame.protocols'].find('http:media') > 0:\n",
    "            proto = 'protocol: http:media'\n",
    "            return \" \".join([src,srcport,proto,size,dst,dstport]).lower()\n",
    "        \n",
    "        \n",
    "        ## icmp:data\n",
    "        ## TODO: Consider adding icmp.type to the frame / protocol language\n",
    "        ## TODO: Detect broadcast\n",
    "        elif line['frame.protocols'].find('icmp:data') > 0:\n",
    "            proto = 'protocol: icmp:data'\n",
    "            return \" \".join([src,proto,size,dst]).lower()\n",
    "        \n",
    "        elif proto == 'arp':\n",
    "            #return \" \".join([src,proto,size,dst])\n",
    "            return \"\".lower()\n",
    "        \n",
    "        ## ethertype:data\n",
    "        elif line['frame.protocols'].find('ethertype:data') > 0:\n",
    "            proto = 'protocol: ethertype:data'\n",
    "            return \" \".join([src,proto,size]).lower()\n",
    "        \n",
    "        # undetected protocols\n",
    "        else:\n",
    "            #is it based on UDP or TCP?\n",
    "            proto = \"protocol: \" + proto\n",
    "            if line['frame.protocols'].find('udp') +\\\n",
    "                line['frame.protocols'].find('tcp') > 0:\n",
    "                return \" \".join([src,srcport,proto,size,dst,dstport]).lower()\n",
    "            elif line['frame.protocols'].find('ip')> 0:\n",
    "                return \" \".join([src,proto,size,dst]).lower()\n",
    "            elif line['frame.protocols'].find('ethertype')> 0:\n",
    "                return \" \".join([src,proto,size,dst]).lower()\n",
    "            else:\n",
    "                print(\"undetected protocol(2): {}\".format(str(line)))\n",
    "                return \" \".join([src,proto,size,dst]).lower()\n",
    "            \n",
    "    except ValueError:\n",
    "        print('Exception!!')\n",
    "        print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocessDF(df):\n",
    "    df['oui.src'] = df['eth.src'].apply(lambda x: ''.join(x.upper().split(':')[0:3]))\n",
    "    df['tail.src'] = df['eth.src'].apply(lambda x: ''.join(x.upper().split(':')[3:6]))\n",
    "    df['oui.dst'] = df['eth.dst'].apply(lambda x: ''.join(x.upper().split(':')[0:3]))\n",
    "    df['tail.dst'] = df['eth.dst'].apply(lambda x: ''.join(x.upper().split(':')[3:6]))\n",
    "    \n",
    "    #create dask frame\n",
    "    df = dd.from_pandas(df, npartitions=8)\n",
    "    \n",
    "    #add OUI\n",
    "    df = df.merge(OUI_df[list(['Assignment', 'Organization Name'])], \n",
    "             left_on = 'oui.src',  right_on= 'Assignment', how = 'left').drop(['Assignment'], axis = 1)\n",
    "    df = df.rename(columns={'Organization Name':'org.src'})\n",
    "\n",
    "    df = df.merge(OUI_df[list(['Assignment', 'Organization Name'])], \n",
    "             left_on = 'oui.dst',  right_on= 'Assignment', how = 'left').drop(['Assignment'], axis = 1)\n",
    "    df = df.rename(columns={'Organization Name':'org.dst'})\n",
    "    \n",
    "    # let's get rid of white spaces\n",
    "    df['org.src'] = df['org.src'].apply(lambda x: x.replace('.','').replace(',','').replace(\" \", \"-\") if pd.notna(x) else x, meta = str)\n",
    "    df['org.dst'] = df['org.dst'].apply(lambda x: x.replace('.','').replace(',','').replace(\" \", \"-\") if pd.notna(x) else x, meta = str)\n",
    "    \n",
    "    # clean up org and tail for dst columns\n",
    "    df['org.dst'] = df['org.dst'].fillna('')\n",
    "    df['tail.dst'] = df[['org.dst','tail.dst','eth.dst_resolved']].apply(lambda row: row['tail.dst'] if (row['org.dst'] != '') \\\n",
    "                                                                     else row['eth.dst_resolved'], axis = 1, meta = str)\n",
    "    # adding private (local) and public (internet) traffic zones\n",
    "    df['zone.src'] = df['ip.src'].apply(lambda x: determineZone(x), meta = str)\n",
    "    df['zone.dst'] = df['ip.dst'].apply(lambda x: determineZone(x), meta = str)\n",
    "    \n",
    "    # clean up IP addresses that have multiple source or dest values\n",
    "    df['ip.src'] = df['ip.src'].apply(lambda x: x.split(',')[0] if pd.notna(x) else x, meta = str)\n",
    "    df['ip.dst'] = df['ip.dst'].apply(lambda x: x.split(',')[0] if pd.notna(x) else x, meta = str)\n",
    "    \n",
    "    # the communication protocol column\n",
    "    df['protocol'] = df['frame.protocols'].apply(lambda x: x.split(':')[-1], meta = str)\n",
    "    \n",
    "    # create a list of public IP addresses that we need to resolve\n",
    "    public_IPs = set()\n",
    "    public_IPs_orgs = {}\n",
    "    for index,row in df.iterrows():\n",
    "        if (row['zone.src'] == \"PUBLIC\"):\n",
    "            public_IPs.add(row['ip.src'])\n",
    "        if (row['zone.dst'] == \"PUBLIC\"):\n",
    "            public_IPs.add(row['ip.dst'])\n",
    "    \n",
    "    # let's look up these public IP addresses in our db and replace with the service name\n",
    "    for ip in public_IPs:\n",
    "        try:\n",
    "            public_IPs_orgs[ip] = ASN_df[(ASN_df['start.dec'] <= float(IPy(ip).strDec())) & \n",
    "                                    (ASN_df['end.dec'] >= float(IPy(ip).strDec()))]['organization']\\\n",
    "                                    .values[0].split(' ')[0]\n",
    "        except:\n",
    "            #print(ip)\n",
    "            public_IPs_orgs[ip] = 'UNLISTED'\n",
    "\n",
    "    # let's replace org data for public IPs based on the ASN information\n",
    "    df['org.src'] = df[['zone.src','ip.src','org.src']].apply(lambda x: ( public_IPs_orgs[x['ip.src']] if x['zone.src'] == \"PUBLIC\" else x['org.src']), axis=1, meta = str)\n",
    "    df['org.dst'] = df[['zone.dst','ip.dst','org.dst']].apply(lambda x: ( public_IPs_orgs[x['ip.dst']] if x['zone.dst'] == \"PUBLIC\" else x['org.dst']), axis=1, meta = str)\n",
    "\n",
    "    # let's create a sentence for each packet\n",
    "    df['sentence'] = df.apply(lambda line: protoLang(line), axis=1, meta = str)\n",
    "    \n",
    "    # Aggregating these sentences for each device is non-trivial. \n",
    "    # read on!\n",
    "    \n",
    "    # Source   Destination\n",
    "    # Private  Private   => L3: local to local \n",
    "    # Private  Public    => L3: local to Internet\n",
    "    # Public   Private   => L3: Internet to Local\n",
    "    # Public   Public    => Shouldn't be possible\n",
    "    # NaN      whatever  => If the source device has no public or private zone (therefore NaN), that means it doesn't have \n",
    "    #                       an IP address, which means it is almost certainly layer 2 (ethernet traffic)\n",
    "\n",
    "    # 3 aggregation\n",
    "    # 1- source = Private (local to local and local to Internet, so to get local devices traffic aggregate by src)\n",
    "    # 2- source = Public  (Internet to local, so to get local devices traffic aggregate by dst)\n",
    "    # 3- source = NA  (L2, the local device can be the receiver or sender)\n",
    "\n",
    "    # L3 traffic from the local network to either the local network or internet\n",
    "    # So aggregate the source local network device communication\n",
    "    df_agg1 = df[df['zone.src'] == 'PRIVATE']\n",
    "    df_agg1 = df_agg1.groupby('eth.src')['sentence'].apply(lambda x: \"\".join(x + (\"\" if len(x) == 3 else \". \")), meta = str).compute()\n",
    "    df_agg1 = pd.Series.to_frame(df_agg1).reset_index()\n",
    "    \n",
    "    # L3 traffic from the public network to the local network (we can't observe Internet to Internet traffic )\n",
    "    # So aggregate the destination local network device communication\n",
    "    df_agg2 = df[df['zone.src'] == 'PUBLIC']\n",
    "    df_agg2 = df_agg2.groupby('eth.dst')['sentence'].apply(lambda x: \"\".join(x + (\"\" if len(x) == 3 else \". \")), meta = str).compute()\n",
    "    df_agg2 = pd.Series.to_frame(df_agg2).reset_index()\n",
    "    \n",
    "    # L2 traffic from the local network to the local network\n",
    "    # A given IoT device can be a source or destination\n",
    "    # so we have to aggregate it in both direction\n",
    "    df_agg3 = df[~df['zone.src'].isin(['PUBLIC','PRIVATE'])]\n",
    "\n",
    "    # aggregate where the IoT device is the source\n",
    "    df_agg3_1 = df_agg3.groupby('eth.src')['sentence'].apply(lambda x: \"\".join(x + (\"\" if len(x) == 3 else \". \")), meta = str).compute()\n",
    "    df_agg3_1 = pd.Series.to_frame(df_agg3_1).reset_index()\n",
    "\n",
    "    # aggregate where the IoT device is the destination\n",
    "    df_agg3_2 = df_agg3.groupby('eth.dst')['sentence'].apply(lambda x: \"\".join(x + (\"\" if len(x) == 3 else \". \")), meta = str).compute()\n",
    "    df_agg3_2 = pd.Series.to_frame(df_agg3_2).reset_index()\n",
    "\n",
    "    final_agg = df_agg1.merge(df_agg2, left_on='eth.src', right_on='eth.dst', how = 'left').drop('eth.dst', axis = 1)\n",
    "    final_agg = final_agg.merge(df_agg3_1, left_on='eth.src', right_on='eth.src', how = 'outer')\n",
    "    final_agg = final_agg.merge(df_agg3_2, left_on='eth.src', right_on='eth.dst', how = 'left').drop('eth.dst', axis = 1)\n",
    "    final_agg.columns = [\"eth\", \"sentence.L3.src\", \"sentence.L3.dst\", \"sentence.L2.src\", \"sentence.L2.dst\"]\n",
    "\n",
    "    final_agg['sentence'] = final_agg.apply(lambda row: \\\n",
    "                            row['sentence.L3.src'] if pd.notna(row['sentence.L3.src']) else \"\" + \\\n",
    "                            row['sentence.L3.dst'] if pd.notna(row['sentence.L3.dst']) else \"\" + \\\n",
    "                            row['sentence.L2.src'] if pd.notna(row['sentence.L2.src']) else \"\" + \\\n",
    "                            row['sentence.L2.dst'] if pd.notna(row['sentence.L2.dst']) else \"\", axis=1)\n",
    "\n",
    "    final_agg = final_agg.drop(['sentence.L3.src','sentence.L3.dst','sentence.L2.src','sentence.L2.dst'], axis = 1)\n",
    "    return final_agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "sydney_1M = pd.read_csv(\"./packets_train_1M.csv\", sep='\\t')\n",
    "sydney_1M_preprocessed = preprocessDF(sydney_1M)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'device withings 1b6f96 port: 68 protocol: udp:bootp size: 342. device withings 1b6f96 port: 68 protocol: udp:bootp size: 342. device withings 1b6f96 port: 68 protocol: udp:bootp size: 342. device withings 1b6f96 port: 49153 protocol: dns size: 80 device tp-link-technologies-coltd 5133ea port: 53 scalews.withings.net . device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 58 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: urlencoded-form size: 237 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: urlencoded-form size: 393 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 253 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 121 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: urlencoded-form size: 516 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: urlencoded-form size: 301 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: urlencoded-form size: 284 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: urlencoded-form size: 512 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 241 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 123 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 330 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 62 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: http:data size: 59 service: withings port: 80 post /cgi-bin/v2/measure withings useragent scalews.withings.net. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: urlencoded-form size: 276 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device withings 1b6f96 port: 68 protocol: udp:bootp size: 342. device withings 1b6f96 port: 68 protocol: udp:bootp size: 342. device withings 1b6f96 port: 68 protocol: udp:bootp size: 342. device withings 1b6f96 port: 68 protocol: udp:bootp size: 342. device withings 1b6f96 port: 49153 protocol: dns size: 80 device tp-link-technologies-coltd 5133ea port: 53 scalews.withings.net . device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 58 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: urlencoded-form size: 237 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: urlencoded-form size: 393 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 253 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 121 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 566 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: urlencoded-form size: 200 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: urlencoded-form size: 301 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: urlencoded-form size: 284 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 241 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 123 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 330 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 333 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 62 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: http:data size: 59 service: withings port: 80 post /cgi-bin/v2/measure withings useragent scalews.withings.net. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: urlencoded-form size: 276 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device: withings identifier: 1b6f96 port: 49153 protocol: tcp size: 54 service: withings port: 80. device withings 1b6f96 port: 68 protocol: udp:bootp size: 342. '"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sydney_1M_preprocessed.iloc[0,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "known_devices = pd.read_csv('sydney_devices.csv')\n",
    "known_devices['OUI'] = known_devices['MAC address'].apply(lambda x: ''.join(x.upper().split(':')[0:3]))\n",
    "known_devices['Manufacturer Device Type'] = ([' '.join(row) for row in \n",
    "                        zip(known_devices[\"Manufacturer\"],known_devices[\"Device Type\"])])\n",
    "known_devices['Manufacturer Device Type'] = known_devices['Manufacturer Device Type'].apply(lambda x: x.strip())\n",
    "known_devices['MAC address'] = known_devices['MAC address'].apply(lambda x: x.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eth</th>\n",
       "      <th>sentence</th>\n",
       "      <th>Manufacturer Device Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00:24:e4:1b:6f:96</td>\n",
       "      <td>device withings 1b6f96 port: 68 protocol: udp:...</td>\n",
       "      <td>Withings Scale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>08:21:ef:3b:fc:e3</td>\n",
       "      <td>device: samsung-electronics-coltd identifier: ...</td>\n",
       "      <td>Samsung Tablet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18:b4:30:25:be:e4</td>\n",
       "      <td>device nest-labs-inc 25bee4 port: 68 protocol:...</td>\n",
       "      <td>Nest Smoke Alarm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00:24:e4:11:18:a8</td>\n",
       "      <td>device: withings identifier: 1118a8 port: 3780...</td>\n",
       "      <td>Withings Baby Monitor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18:b7:9e:02:20:44</td>\n",
       "      <td>device: invoxia identifier: 022044 port: 40234...</td>\n",
       "      <td>Triby Speaker</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>44:65:0d:56:cc:d3</td>\n",
       "      <td>device amazon-technologies-inc 56ccd3 port: 60...</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>74:2f:68:81:69:42</td>\n",
       "      <td>device azurewave-technology-inc 816942 port: 6...</td>\n",
       "      <td>Azure Laptop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>f4:f2:6d:93:51:f1</td>\n",
       "      <td>device: tp-link-technologies-coltd identifier:...</td>\n",
       "      <td>TP-Link Camera</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>14:cc:20:51:33:ea</td>\n",
       "      <td>device tp-link-technologies-coltd 5133ea port:...</td>\n",
       "      <td>TP-Link Router</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30:8c:fb:2f:e4:b2</td>\n",
       "      <td>device: dropcam identifier: 2fe4b2 port: 46330...</td>\n",
       "      <td>Dropcam Camera</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>50:c7:bf:00:56:39</td>\n",
       "      <td>device: tp-link-technologies-coltd identifier:...</td>\n",
       "      <td>TP-Link Smart Plug</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ec:1a:59:83:28:11</td>\n",
       "      <td>device: belkin-international-inc identifier: 8...</td>\n",
       "      <td>Belkin Wemo Sensor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>d0:a6:37:df:a1:e1</td>\n",
       "      <td>device apple-inc dfa1e1 port: 68 protocol: udp...</td>\n",
       "      <td>Apple IPhone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ec:1a:59:79:f4:89</td>\n",
       "      <td>device: belkin-international-inc identifier: 7...</td>\n",
       "      <td>Belkin Wemo Switch</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>70:ee:50:03:b8:ac</td>\n",
       "      <td>device netatmo 03b8ac port: 68 protocol: udp:b...</td>\n",
       "      <td>Netatmo Weather Station</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>70:ee:50:18:34:43</td>\n",
       "      <td>device netatmo 183443 port: 61725 protocol: dn...</td>\n",
       "      <td>Netatmo Welcome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>74:6a:89:00:2e:25</td>\n",
       "      <td>device rezolt-corporation 002e25 port: 68 prot...</td>\n",
       "      <td>Blipcare Blood Pressure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>b4:ce:f6:a7:a3:c2</td>\n",
       "      <td>device htc-corporation a7a3c2 port: 68 protoco...</td>\n",
       "      <td>Android Phone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>e0:76:d0:33:bb:85</td>\n",
       "      <td>device ampak-technology-inc 33bb85 port: 138 p...</td>\n",
       "      <td>PIX-STAR Photo Frame</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>00:16:6c:ab:6b:88</td>\n",
       "      <td>device samsung-electronics-coltd ab6b88 port: ...</td>\n",
       "      <td>Samsung Camera</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>70:5a:0f:e4:9b:c0</td>\n",
       "      <td>device hewlett-packard e49bc0 protocol: igmp ....</td>\n",
       "      <td>HP Printer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>d0:52:a8:00:67:5e</td>\n",
       "      <td>device: physical-graph-corporation identifier:...</td>\n",
       "      <td>Samsung Smart Things</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  eth                                           sentence  \\\n",
       "0   00:24:e4:1b:6f:96  device withings 1b6f96 port: 68 protocol: udp:...   \n",
       "1   08:21:ef:3b:fc:e3  device: samsung-electronics-coltd identifier: ...   \n",
       "2   18:b4:30:25:be:e4  device nest-labs-inc 25bee4 port: 68 protocol:...   \n",
       "3   00:24:e4:11:18:a8  device: withings identifier: 1118a8 port: 3780...   \n",
       "4   18:b7:9e:02:20:44  device: invoxia identifier: 022044 port: 40234...   \n",
       "5   44:65:0d:56:cc:d3  device amazon-technologies-inc 56ccd3 port: 60...   \n",
       "6   74:2f:68:81:69:42  device azurewave-technology-inc 816942 port: 6...   \n",
       "7   f4:f2:6d:93:51:f1  device: tp-link-technologies-coltd identifier:...   \n",
       "8   14:cc:20:51:33:ea  device tp-link-technologies-coltd 5133ea port:...   \n",
       "9   30:8c:fb:2f:e4:b2  device: dropcam identifier: 2fe4b2 port: 46330...   \n",
       "10  50:c7:bf:00:56:39  device: tp-link-technologies-coltd identifier:...   \n",
       "11  ec:1a:59:83:28:11  device: belkin-international-inc identifier: 8...   \n",
       "12  d0:a6:37:df:a1:e1  device apple-inc dfa1e1 port: 68 protocol: udp...   \n",
       "13  ec:1a:59:79:f4:89  device: belkin-international-inc identifier: 7...   \n",
       "14  70:ee:50:03:b8:ac  device netatmo 03b8ac port: 68 protocol: udp:b...   \n",
       "15  70:ee:50:18:34:43  device netatmo 183443 port: 61725 protocol: dn...   \n",
       "16  74:6a:89:00:2e:25  device rezolt-corporation 002e25 port: 68 prot...   \n",
       "17  b4:ce:f6:a7:a3:c2  device htc-corporation a7a3c2 port: 68 protoco...   \n",
       "18  e0:76:d0:33:bb:85  device ampak-technology-inc 33bb85 port: 138 p...   \n",
       "19  00:16:6c:ab:6b:88  device samsung-electronics-coltd ab6b88 port: ...   \n",
       "20  70:5a:0f:e4:9b:c0  device hewlett-packard e49bc0 protocol: igmp ....   \n",
       "21  d0:52:a8:00:67:5e  device: physical-graph-corporation identifier:...   \n",
       "\n",
       "   Manufacturer Device Type  \n",
       "0            Withings Scale  \n",
       "1            Samsung Tablet  \n",
       "2          Nest Smoke Alarm  \n",
       "3     Withings Baby Monitor  \n",
       "4             Triby Speaker  \n",
       "5               Amazon Echo  \n",
       "6              Azure Laptop  \n",
       "7            TP-Link Camera  \n",
       "8            TP-Link Router  \n",
       "9            Dropcam Camera  \n",
       "10       TP-Link Smart Plug  \n",
       "11       Belkin Wemo Sensor  \n",
       "12             Apple IPhone  \n",
       "13       Belkin Wemo Switch  \n",
       "14  Netatmo Weather Station  \n",
       "15          Netatmo Welcome  \n",
       "16  Blipcare Blood Pressure  \n",
       "17            Android Phone  \n",
       "18     PIX-STAR Photo Frame  \n",
       "19           Samsung Camera  \n",
       "20               HP Printer  \n",
       "21     Samsung Smart Things  "
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sydney_1M_preprocessed = sydney_1M_preprocessed.merge(known_devices[['MAC address','Manufacturer Device Type']], \n",
    "                left_on='eth', right_on='MAC address', how = 'left').drop('MAC address', axis = 1)\n",
    "sydney_1M_preprocessed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sydney_1M_preprocessed.to_csv(\"sydney_1M_preprocessed.csv\", header=None, index=None, sep=' ')\n",
    "sydney_1M_preprocessed.columns = ['eth','sentence','label']\n",
    "# prune out empty sentences\n",
    "sydney_1M_preprocessed = sydney_1M_preprocessed.loc[sydney_1M_preprocessed['sentence'] != '']\n",
    "# write out to a csv file for the next round\n",
    "sydney_1M_preprocessed.to_csv(\"sydney_1M_preprocessed.csv\",index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "sydney_1M_embeddings = sydney_1M_preprocessed.sentence.to_frame()\n",
    "sydney_1M_embeddings.to_csv('sydney_1M_embeddings.txt', \n",
    "                            header=None, index=None, sep=' ', quoting = csv.QUOTE_NONE, escapechar=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! head -n 1 sydney_1M_embeddings.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparing test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eth</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00:18:0a:12:2f:2d</td>\n",
       "      <td>device: cisco-meraki identifier: 122f2d port: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b8:e9:37:8c:da:3c</td>\n",
       "      <td>device: sonos-inc identifier: 8cda3c port: 413...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28:f0:76:31:d3:58</td>\n",
       "      <td>device: apple-inc identifier: 31d358 port: 654...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00:18:0a:7d:01:ce</td>\n",
       "      <td>device cisco-meraki 7d01ce protocol: icmp:data...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>40:cb:c0:bc:36:7e</td>\n",
       "      <td>device apple-inc bc367e port: 5353 protocol: m...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 eth                                           sentence\n",
       "0  00:18:0a:12:2f:2d  device: cisco-meraki identifier: 122f2d port: ...\n",
       "1  b8:e9:37:8c:da:3c  device: sonos-inc identifier: 8cda3c port: 413...\n",
       "2  28:f0:76:31:d3:58  device: apple-inc identifier: 31d358 port: 654...\n",
       "3  00:18:0a:7d:01:ce  device cisco-meraki 7d01ce protocol: icmp:data...\n",
       "4  40:cb:c0:bc:36:7e  device apple-inc bc367e port: 5353 protocol: m..."
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "daghan_df = pd.read_csv(\"./daghan.csv\", sep='\\t')\n",
    "daghan_preprocessed = preprocessDF(daghan_df)\n",
    "daghan_preprocessed.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "known_devices = pd.read_csv('known_devices.csv')\n",
    "known_devices['OUI'] = known_devices['MAC address'].apply(lambda x: ''.join(x.upper().split(':')[0:3]))\n",
    "known_devices['Manufacturer Device Type'] = ([' '.join(row) for row in \n",
    "                        zip(known_devices[\"Manufacturer\"],known_devices[\"Device Type\"])])\n",
    "known_devices['Manufacturer Device Type'] = known_devices['Manufacturer Device Type'].apply(lambda x: x.strip())\n",
    "known_devices['MAC address'] = known_devices['MAC address'].apply(lambda x: x.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "daghan_preprocessed = daghan_preprocessed.merge(known_devices[['MAC address','Manufacturer Device Type']], \n",
    "                left_on='eth', right_on='MAC address', how = 'left').dropna().drop('MAC address', axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "daghan_preprocessed.columns = ['eth','sentence','label']\n",
    "daghan_preprocessed = daghan_preprocessed.loc[daghan_preprocessed['sentence'] != '']\n",
    "daghan_preprocessed.to_csv(\"daghan_preprocessed.csv\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eth</th>\n",
       "      <th>sentence</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00:18:0a:12:2f:2d</td>\n",
       "      <td>device: cisco-meraki identifier: 122f2d port: ...</td>\n",
       "      <td>Meraki Phone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b8:e9:37:8c:da:3c</td>\n",
       "      <td>device: sonos-inc identifier: 8cda3c port: 413...</td>\n",
       "      <td>Sonos Speaker</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28:f0:76:31:d3:58</td>\n",
       "      <td>device: apple-inc identifier: 31d358 port: 654...</td>\n",
       "      <td>Apple Computer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>40:cb:c0:bc:36:7e</td>\n",
       "      <td>device apple-inc bc367e port: 5353 protocol: m...</td>\n",
       "      <td>Apple TV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>44:65:0d:90:60:3a</td>\n",
       "      <td>device amazon-technologies-inc 90603a port: 55...</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>78:28:ca:32:4b:28</td>\n",
       "      <td>device sonos-inc 324b28 port: 43082 protocol: ...</td>\n",
       "      <td>Sonos Speaker</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>38:c9:86:40:7c:a6</td>\n",
       "      <td>device apple-inc 407ca6 protocol: icmp:data si...</td>\n",
       "      <td>Apple Computer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>84:38:35:5a:70:40</td>\n",
       "      <td>device: apple-inc identifier: 5a7040 port: 595...</td>\n",
       "      <td>Apple Computer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>5c:aa:fd:4c:87:a0</td>\n",
       "      <td>device: sonos-inc identifier: 4c87a0 port: 446...</td>\n",
       "      <td>Sonos Speaker</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>78:28:ca:03:80:0c</td>\n",
       "      <td>device: sonos-inc identifier: 03800c port: 570...</td>\n",
       "      <td>Sonos Speaker</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>b4:7c:9c:31:b4:df</td>\n",
       "      <td>device amazon-technologies-inc 31b4df protocol...</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>5c:aa:fd:4c:92:86</td>\n",
       "      <td>device: sonos-inc identifier: 4c9286 port: 336...</td>\n",
       "      <td>Sonos Speaker</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>88:71:e5:d2:73:4b</td>\n",
       "      <td>device: amazon-technologies-inc identifier: d2...</td>\n",
       "      <td>Amazon Echo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ec:1a:59:f6:eb:45</td>\n",
       "      <td>device: belkin-international-inc identifier: f...</td>\n",
       "      <td>Belkin Wemo switch</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  eth                                           sentence  \\\n",
       "0   00:18:0a:12:2f:2d  device: cisco-meraki identifier: 122f2d port: ...   \n",
       "1   b8:e9:37:8c:da:3c  device: sonos-inc identifier: 8cda3c port: 413...   \n",
       "2   28:f0:76:31:d3:58  device: apple-inc identifier: 31d358 port: 654...   \n",
       "4   40:cb:c0:bc:36:7e  device apple-inc bc367e port: 5353 protocol: m...   \n",
       "5   44:65:0d:90:60:3a  device amazon-technologies-inc 90603a port: 55...   \n",
       "6   78:28:ca:32:4b:28  device sonos-inc 324b28 port: 43082 protocol: ...   \n",
       "8   38:c9:86:40:7c:a6  device apple-inc 407ca6 protocol: icmp:data si...   \n",
       "10  84:38:35:5a:70:40  device: apple-inc identifier: 5a7040 port: 595...   \n",
       "12  5c:aa:fd:4c:87:a0  device: sonos-inc identifier: 4c87a0 port: 446...   \n",
       "13  78:28:ca:03:80:0c  device: sonos-inc identifier: 03800c port: 570...   \n",
       "14  b4:7c:9c:31:b4:df  device amazon-technologies-inc 31b4df protocol...   \n",
       "15  5c:aa:fd:4c:92:86  device: sonos-inc identifier: 4c9286 port: 336...   \n",
       "16  88:71:e5:d2:73:4b  device: amazon-technologies-inc identifier: d2...   \n",
       "17  ec:1a:59:f6:eb:45  device: belkin-international-inc identifier: f...   \n",
       "\n",
       "                 label  \n",
       "0         Meraki Phone  \n",
       "1        Sonos Speaker  \n",
       "2       Apple Computer  \n",
       "4             Apple TV  \n",
       "5          Amazon Echo  \n",
       "6        Sonos Speaker  \n",
       "8       Apple Computer  \n",
       "10      Apple Computer  \n",
       "12       Sonos Speaker  \n",
       "13       Sonos Speaker  \n",
       "14         Amazon Echo  \n",
       "15       Sonos Speaker  \n",
       "16         Amazon Echo  \n",
       "17  Belkin Wemo switch  "
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "daghan_preprocessed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}