main_handout.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "# 所需套件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "import jieba\n",
    "import jieba.posseg as pseg\n",
    "import jieba.analyse\n",
    "\n",
    "import speech_recognition as sr\n",
    "from pprint import pprint\n",
    "from os import path\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import requests as req\n",
    "import os\n",
    "from xml.etree import ElementTree\n",
    "import pyaudio\n",
    "import wave\n",
    "import sys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## 辨識使用者說什麼，轉成文字"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "def listen_user_says(my_key):\n",
    "    r = sr.Recognizer()\n",
    "    with sr.Microphone() as source:\n",
    "        print ('Say Something!')\n",
    "        audio = r.listen(source)\n",
    "        print ('Done!')\n",
    "    try:\n",
    "        print(\"Microsoft Bing Voice Recognition thinks you said: \")\n",
    "        # pprint(r.recognize_bing(audio, key=my_key, language=\"zh-CN\", show_all=True))\n",
    "        return r.recognize_bing(audio, key=my_key, language=\"zh-TW\") \n",
    "    except sr.UnknownValueError:\n",
    "        print(\"Microsoft Bing Voice Recognition could not understand audio\")\n",
    "    except sr.RequestError as e:\n",
    "        print(\"Could not request results from Microsoft Bing Voice Recognition service; {0}\".format(e))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## 傳入使用者所說的文字，進行分詞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "def split_by_Jieba(text):\n",
    "    jieba.set_dictionary(\"dict/dict.txt.big.txt\")\n",
    "    jieba.load_userdict(\"dict/userdict.txt\")\n",
    "    \n",
    "    simple_list = []\n",
    "    simple_list = list(jieba.cut(text))\n",
    "    return simple_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## 針對主題進行爬蟲"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "def scrape_by_theme(theme):\n",
    "    base_url = \"https://udn.com\"\n",
    "    start_url = base_url + \"/news/index\"\n",
    "    req = requests.get(start_url)\n",
    "    soupObj = BeautifulSoup(req.text, 'html.parser')\n",
    "    sports=\"\"\n",
    "\n",
    "    print(\"in scraper, the theme is : \" + theme)\n",
    "    for link in soupObj.find_all('a'):\n",
    "        s = set(link.contents)\n",
    "        if theme in s:\n",
    "            temp = link.get('href')\n",
    "            sports = temp\n",
    "            break\n",
    "\n",
    "    sports_url = base_url + sports\n",
    "    \n",
    "    print(sports_url)\n",
    "    req2 = requests.get(sports_url)\n",
    "    print(req2.status_code)\n",
    "    soupObj2 = BeautifulSoup(req2.text, 'html.parser')\n",
    "\n",
    "    ob = soupObj2.find_all(\"div\", attrs={'class': 'area'})\n",
    "    top5_news = []\n",
    "\n",
    "    for i in ob:\n",
    "        x = i.find_next(\"h3\")\n",
    "        print(x.contents)\n",
    "        if u\"最新文章\" in x.contents:\n",
    "            top_news = x.next_sibling\n",
    "            for item in top_news.find_all(\"h2\"):\n",
    "                top5_news.append(item.get_text())\n",
    "            break\n",
    "    return top5_news "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def toSound(news_list,api_key):\n",
    "\n",
    "    headers = {'Ocp-Apim-Subscription-Key': api_key}\n",
    "    get_token = req.session()\n",
    "    access_token = get_token.post('https://api.cognitive.microsoft.com/sts/v1.0/issueToken', headers=headers).text\n",
    "\n",
    "    headers = {\"Content-type\": \"application/ssml+xml\",\n",
    "            \"X-Microsoft-OutputFormat\": \"riff-16khz-16bit-mono-pcm\",\n",
    "            \"Authorization\": \"Bearer \" + access_token}\n",
    "\n",
    "    # Changing voice output via SSML\n",
    "    body = ElementTree.Element('speak', version='1.0')\n",
    "    body.set('xml:lang', 'en-GB')\n",
    "    voice = ElementTree.SubElement(body, 'voice')\n",
    "    voice.set('xml:lang', 'en-GB')\n",
    "    voice.set('xml:gender', 'Female')\n",
    "\n",
    "    voice.set('name', 'Microsoft Server Speech Text to Speech Voice (zh-TW, Yating, Apollo)')\n",
    "\n",
    "    # 建立資料夾\n",
    "    get_record = req.session()\n",
    "    os.makedirs('./output', exist_ok=True)\n",
    "    \n",
    "\n",
    "    i = 0\n",
    "    for text in news_list:\n",
    "        i+=1\n",
    "        voice.text = text\n",
    "        respond = get_record.post('https://speech.platform.bing.com/synthesize', data=ElementTree.tostring(body), headers=headers)\n",
    "        \n",
    "        audio_name = \"news_\"+ str(i)+ \".wav\"\n",
    "        with open('./output/' + audio_name, 'wb') as f:\n",
    "            f.write(respond.content)\n",
    "\n",
    "        # play the sound we just turn in\n",
    "\n",
    "        CHUNK = 1024\n",
    "\n",
    "        wf = wave.open(\"./output/\"+audio_name, 'rb')\n",
    "\n",
    "        p = pyaudio.PyAudio()\n",
    "\n",
    "        stream = p.open(format=p.get_format_from_width(wf.getsampwidth()),\n",
    "                        channels=wf.getnchannels(),\n",
    "                        rate=wf.getframerate(),\n",
    "                        output=True)\n",
    "\n",
    "        data = wf.readframes(CHUNK)\n",
    "\n",
    "        while len(data) > 0:\n",
    "            stream.write(data)\n",
    "            data = wf.readframes(CHUNK)\n",
    "\n",
    "        stream.stop_stream()\n",
    "        stream.close()\n",
    "\n",
    "        p.terminate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 來跑跑看吧～"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Say Something!\n",
      "Done!\n",
      "Microsoft Bing Voice Recognition thinks you said: \n",
      "我想知道運動的新聞\n"
     ]
    }
   ],
   "source": [
    "# your key here\n",
    "BING_KEY = \"\"\n",
    "\n",
    "# 使用 bing speech api 音檔轉文字的服務\n",
    "user_says = listen_user_says(BING_KEY)\n",
    "print(user_says)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# jieba 分詞\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from /Users/tp6han/Documents/python/scraper_bing_speech_api/dict/dict.txt.big.txt ...\n",
      "Loading model from cache /var/folders/sl/zps8bzs557qgnvccrm1wtbx00000gn/T/jieba.u2386d3d8b4957de4a74d0f468bedc473.cache\n",
      "Loading model cost 1.503 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['我', '想', '知道', '全球', '的', '新聞']\n"
     ]
    }
   ],
   "source": [
    "sep_words = split_by_Jieba(user_says)\n",
    "print(sep_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /var/folders/sl/zps8bzs557qgnvccrm1wtbx00000gn/T/jieba.cache\n",
      "Loading model cost 0.907 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['運動']\n"
     ]
    }
   ],
   "source": [
    "tags = jieba.analyse.extract_tags(user_says, topK=1)\n",
    "print(tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "產經\n"
     ]
    }
   ],
   "source": [
    "print('產經')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "運動\n"
     ]
    }
   ],
   "source": [
    "theme = str(tags[0])\n",
    "print(theme)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sep_words' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-46-ad7cae4f6dd3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mitem\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msep_words\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"item >>> \"\u001b[0m\u001b[0;34m+\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mitem\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34mu'運動'\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mitem\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34mu'體育'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0mtheme\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mu'運動'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"user want sports news\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sep_words' is not defined"
     ]
    }
   ],
   "source": [
    "for item in sep_words:\n",
    "    print(\"item >>> \"+ item)\n",
    "    if item == u'運動' or item == u'體育':\n",
    "        theme = u'運動'\n",
    "        print(\"user want sports news\")\n",
    "    elif item == u'股市':\n",
    "        theme = u'股市'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in scraper, the theme is : 運動\n",
      "https://udn.com/news/cate/2/7227\n",
      "200\n",
      "['運動頭條']\n",
      "['最新文章', <a class=\"more\" href=\"/news/breaknews/1/7#breaknews\" target=\"_blank\" title=\"more\"></a>]\n",
      ">>>>>   in main\n",
      "田徑／相隔15年 吳佳如「撐」破全國\n",
      "跆拳道／羅嘉翎「金」喜 世青賽連兩屆摘金\n",
      "傷兵多、黑馬強 引出季後賽首輪三大話題\n",
      "籃球／理查森替NBA 3x造勢 預測勇士衛冕成功\n",
      "LPGA／樂天高球賽首輪 錢珮芸並列49名台將最佳\n"
     ]
    }
   ],
   "source": [
    "top5_news = scrape_by_theme(theme)\n",
    "print(*top5_news, sep='\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['田徑／相隔15年 吳佳如「撐」破全國',\n",
       " '跆拳道／羅嘉翎「金」喜 世青賽連兩屆摘金',\n",
       " '傷兵多、黑馬強 引出季後賽首輪三大話題',\n",
       " '籃球／理查森替NBA 3x造勢 預測勇士衛冕成功',\n",
       " 'LPGA／樂天高球賽首輪 錢珮芸並列49名台將最佳']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top5_news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "toSound(top5_news,BING_KEY)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 運動 股市"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}