main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "from selenium.webdriver.support.wait import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "import os\n",
    "from time import sleep\n",
    "import student_info\n",
    "from bb_info import *\n",
    "\n",
    "while True:\n",
    "    subject = input(\"PLease provide a subject name as the folder (Ex: EIE2001):\")\n",
    "    try:\n",
    "        folder_location = os.path.join(os.getcwd(), subject)\n",
    "        if not os.path.exists(folder_location):\n",
    "            os.mkdir(folder_location)\n",
    "        break\n",
    "    except Exception as e:\n",
    "        print(f\"{subject} is not a valid folder name, please modify\")\n",
    "        continue\n",
    "\n",
    "# Configuration for browser to download automatically and to a specific directory\n",
    "options = webdriver.ChromeOptions()\n",
    "chrome_prefs = {\n",
    "    \"download.prompt_for_download\": False,\n",
    "    \"plugins.always_open_pdf_externally\": True,\n",
    "    \"download.open_pdf_in_system_reader\": False,\n",
    "    \"profile.default_content_settings.popups\": 0,\n",
    "    \"download.default_directory\": folder_location\n",
    "}\n",
    "options.add_experimental_option(\"prefs\", chrome_prefs)\n",
    "\n",
    "# Note: require chromedriver.exe in path, you may put it in directory with the script\n",
    "driver = webdriver.Chrome(options=options)\n",
    "\n",
    "# BB's login page\n",
    "driver.get(base_url)\n",
    "# Enter student ID and password and login\n",
    "# input student id\n",
    "driver.find_element(By.XPATH, input_SID_xpath).send_keys(student_info.S_ID)\n",
    "# input password\n",
    "driver.find_element(By.XPATH, \n",
    "    input_password_xpath).send_keys(student_info.password)\n",
    "# Login\n",
    "driver.find_element(By.XPATH, login_button_xpath).send_keys(Keys.ENTER)\n",
    "\n",
    "# cookie agreement\n",
    "WebDriverWait(driver, 10).until(\n",
    "    EC.presence_of_element_located(\n",
    "        (By.XPATH, privacy_cookie_page_button_bb_xpath)\n",
    "    )\n",
    ")\n",
    "driver.find_element(By.XPATH, \n",
    "    privacy_cookie_page_button_bb_xpath).send_keys(Keys.ENTER)\n",
    "\n",
    "# Go to course page\n",
    "driver.get(base_url+coures_page)\n",
    "# Now manually go to the target page that you would like to start your download"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Manually go to a main content page of a course"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IS_TEST = False  # This only works for CSC3100 2022-2023 Fall Term course\n",
    "# if not please manually go to the Main content page\n",
    "# An example url for main content page is below\n",
    "# https://bb.cuhk.edu.cn/webapps/blackboard/content/listContent.jsp?course_id=_8275_1&content_id=_274527_1&mode=reset\n",
    "if IS_TEST:\n",
    "    course_page = \"webapps/blackboard/execute/modulepage/view?course_id={course}&cmp_tab_id={cmp_tab}\"\n",
    "    course_content_page = \"webapps/blackboard/content/listContent.jsp?course_id={course}&content_id={content}\"\n",
    "    course_id = \"_8275_1\"  # Not useful, it is course specific one\n",
    "    cmp_tab_id = '_17861_1'\n",
    "    main_content_id = '_274527_1'\n",
    "    # This may needs automation\n",
    "    my_url = base_url + \\\n",
    "        course_content_page.format(course=course_id, content=main_content_id)\n",
    "    driver.get(my_url)\n",
    "else:\n",
    "    input(\"Is the browser at the content page? then press Enter \")\n",
    "\n",
    "\n",
    "def save_text_in_page(file: str = \"Obtain_text.txt\", title=\"\"):\n",
    "    # Obtain page title\n",
    "    if title == \"\":\n",
    "        try:\n",
    "            title = driver.find_element(By.ID, \"pageTitleText\").text\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            pass\n",
    "    else:\n",
    "        title = \"page_title\"\n",
    "    # begin writing to file\n",
    "    with open(file, 'a') as f:\n",
    "        f.write(\"-\"*5 + title + \"-\"*5 + \"\\n\")\n",
    "\n",
    "    # Scrape plain text from contentlist\n",
    "    try:\n",
    "        content_list = driver.find_element(By.ID, \"content_listContainer\")\n",
    "        text_elements = content_list.find_elements(By.CLASS_NAME, \"read\")\n",
    "        for ele in text_elements:\n",
    "            with open(file, 'a') as f:\n",
    "                f.write(\"\\t\" + ele.text.replace(\"\\n\", \"    \") + '\\n')\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "    # Ending\n",
    "    with open(file, 'a') as f:\n",
    "        f.write(\"=\"*5 + f\"END {title}\" + \"=\"*5 + \"\\n\")\n",
    "\n",
    "\n",
    "def download_files_from_this_page(total_file_links=[], pass_folder_links=[], depth=0, max_depth=-1):\n",
    "    if max_depth >= 0 and depth > max_depth:\n",
    "        return\n",
    "    # locate file links in this page\n",
    "    try:\n",
    "        content_list = driver.find_element(By.CLASS_NAME, \"contentList\")\n",
    "        page_urls = content_list.find_elements(By.TAG_NAME, 'a')\n",
    "\n",
    "        # print(len(set(page_urls)) == len(page_urls)) # whether all urls in this page are unique, seems always True\n",
    "\n",
    "        for url in page_urls:\n",
    "            page_url = url.get_attribute('href')\n",
    "            if 'bbcswebdav' in page_url and page_url not in total_file_links:\n",
    "                total_file_links.append(page_url)\n",
    "                print(\"find new file link: \", page_url)\n",
    "    except NoSuchElementException:\n",
    "        # print(\"Content list not found in {driver.current_url}\")\n",
    "        pass\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "    pass_folder_links.append(driver.current_url)\n",
    "    # locate each folder\n",
    "    folder_links = []\n",
    "    try:  # avoid stale references errors?\n",
    "        folders = driver.find_element(\n",
    "            By.XPATH, folder_entry_xpath).find_elements(By.CLASS_NAME, \"read\")\n",
    "        for ele in folders:\n",
    "            try:\n",
    "                ref_link = ele.find_element(\n",
    "                    By.TAG_NAME, 'a').get_attribute('href')\n",
    "                if content_url in ref_link and ref_link not in folder_links and ref_link not in pass_folder_links:\n",
    "                    folder_links.append(ref_link)\n",
    "                    print(\"Add folder link\", ref_link)\n",
    "                else:\n",
    "                    # print(f\"{ref_link} is not a folder link in bb\")\n",
    "                    pass\n",
    "            except NoSuchElementException:\n",
    "                # print(f\"No element found in {ele.text}\")\n",
    "                continue\n",
    "    except:\n",
    "        pass\n",
    "    # go through each folder\n",
    "    for url in folder_links:\n",
    "        driver.get(url)\n",
    "        pass_folder_links.append(url)\n",
    "        print(\"Enter folder link: \", url)\n",
    "        download_files_from_this_page(\n",
    "            total_file_links, pass_folder_links, depth+1)\n",
    "\n",
    "    if depth == 0:\n",
    "        with open('file_links.txt', 'a') as f:\n",
    "            for url in total_file_links:\n",
    "                f.write(url + \"\\n\")\n",
    "\n",
    "        # download all file in file_links\n",
    "        print(\"-\"*10, \"download starts\", \"-\"*10)\n",
    "        for idx, url in enumerate(total_file_links):\n",
    "            if idx % 5 == 0:\n",
    "                sleep(5)  # No meaning, please ignore\n",
    "            driver.get(url)\n",
    "        print(\"-\"*10, \"download has finished\", \"-\"*10)\n",
    "\n",
    "\n",
    "file_links = []\n",
    "passed_folder_links = []\n",
    "depth = 0  # download collected files at depth 0\n",
    "max_depth = -1 # no maximum depth\n",
    "download_files_from_this_page(\n",
    "    total_file_links=file_links, pass_folder_links=passed_folder_links, depth=depth, max_depth=max_depth\n",
    "    )\n",
    "\n",
    "# obtain plain text info from each page\n",
    "text_file = f\"{subject}/{subject}_scraped_text.txt\"\n",
    "for link in passed_folder_links:\n",
    "    driver.get(link)\n",
    "    save_text_in_page(file=text_file)\n",
    "# All has finished here, you may run next blcok or simply shutdown the script if your download is satisfied\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.close()  # Everything done!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Below are useless only for my testing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "content_ele = driver.find_element(By.XPATH, folder_entry_xpath)\n",
    "print(\"--\"*5 + \"possible folder names \" + \"--\"*5)\n",
    "print(content_ele.text)\n",
    "\n",
    "# Obtain each folder's reference link\n",
    "# folders = driver.find_elements(By.CLASS_NAME, \"read\")\n",
    "folders = content_ele.find_elements(By.CLASS_NAME, \"read\")\n",
    "\n",
    "folder_links = []\n",
    "ref_link = \"\"\n",
    "\n",
    "for element in folders:\n",
    "    try:\n",
    "        a_tag = element.find_element(By.TAG_NAME, 'a')\n",
    "    except NoSuchElementException:\n",
    "        print(f\"No element found in {element.text}\")\n",
    "        continue\n",
    "    ref_link = a_tag.get_attribute('href')\n",
    "    if content_url in ref_link:\n",
    "        folder_links.append(ref_link)\n",
    "    else:\n",
    "        print(f\"{ref_link} is not a folder link in bb\")\n",
    "print(\"--\"*5 + \"Folder links\" + \"--\"*5)\n",
    "print(folder_links)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Obtain download urls from current page\n",
    "file_links = []\n",
    "try:\n",
    "    content_list = driver.find_element(By.CLASS_NAME, \"contentList\")\n",
    "    page_urls = content_list.find_elements(By.TAG_NAME, 'a')\n",
    "\n",
    "    print(len(set(page_urls)) == len(page_urls))\n",
    "    with open(\"file_link.txt\", 'w') as f:\n",
    "        with open('other_link.txt', 'w') as f2:\n",
    "            for url in page_urls:\n",
    "                page_url = url.get_attribute('href')\n",
    "                if 'bbcswebdav' in page_url and page_url not in file_links:\n",
    "                    file_links.append(page_url)\n",
    "                    f.write(page_url + '\\n')\n",
    "                else:\n",
    "                    f2.write(page_url + '\\n')\n",
    "except NoSuchElementException:\n",
    "    print(\"Content list not found in {driver.current_url}\")\n",
    "\n",
    "\n",
    "# for url in page_urls:\n",
    "#     print(url.get_attribute('href'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "real_file_links = []\n",
    "\n",
    "with open('pdf_link.txt', 'w') as f:\n",
    "\n",
    "    for url in file_links:\n",
    "        driver.get(url)\n",
    "        real_file_links.append(driver.current_url)\n",
    "        print(driver.current_url)\n",
    "        f.write(driver.current_url + '\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download pdf from url to specific folders https://scripteverything.com/download-pdf-selenium-python/\n",
    "js_script = f''' \n",
    "\t\t\tbody = document.querySelector('body');\n",
    "\t\t\telement = document.createElement('a');\n",
    "            element.href = \"{real_file_links[5]}\"\n",
    "            element.download = \"\";\n",
    "\t\t\ttext = document.createTextNode('Hello WOrld YABINYABIN');\n",
    "\t\t\telement.appendChild(text);\n",
    "\t\t\tbody.append(element);\n",
    "            //element.click()\n",
    "'''\n",
    "\n",
    "driver.execute_script(js_script)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "options = webdriver.ChromeOptions()\n",
    "options.add_experimental_option('prefs', {\n",
    "    # Change default directory for downloads\n",
    "    \"download.default_directory\": \"C:/Users/XXXX/Desktop\",\n",
    "    \"download.prompt_for_download\": False,  # To auto download the file\n",
    "    \"download.directory_upgrade\": True,\n",
    "    # It will not show PDF directly in chrome\n",
    "    \"plugins.always_open_pdf_externally\": True\n",
    "})\n",
    "driver.create_options()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "driver.get(\n",
    "    'https://bb.cuhk.edu.cn/bbcswebdav/pid-317351-dt-content-rid-4972414_1/xid-4972414_1')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for url in folder_links:\n",
    "    driver.get(url)\n",
    "    sleep(20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.close()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "cd882dfa16736b50d668d4c167b4d759d996ae1bea110bbbc05da598ba0e86f2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}