amazon_price_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def split(txt):\n",
    "    txt = txt.split(\"$\", 2)\n",
    "    if txt:\n",
    "        txt = str(\"$\" + str(txt[1]))\n",
    "    return txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.amazon.com/DOG-CARE-Vibration-Waterproof-Rechargeable/dp/B0BM9R8TS9?ref_=Oct_DLandingS_D_2f0a1271_0\n",
      "200\n",
      "Downloading https://www.amazon.com/Adam-Lippes-Collective-Design-Desprit/dp/B0B9R7QL1T?ref_=Oct_DLandingS_D_c69acd91_0&th=1&psc=1\n",
      "200\n",
      "Downloading https://www.amazon.com/dp/B08C1W5N87?ref_=Oct_DLandingS_D_de598b05_NA\n",
      "200\n",
      "Downloading https://www.amazon.com/dp/B07W5JZBVJ?ref_=Oct_DLandingS_D_8268d41f_NA\n",
      "200\n",
      "Downloading https://www.amazon.com/dp/B0C818CYK7/ref=sr_1_1?crid=M44QA5QQUPOO\n",
      "200\n",
      "Downloading https://www.amazon.com/KIIU-Indoor-Outdoor-Sandals-Slippers/dp/B07W92HWDP/ref=sr_1_1_sspa?adgrpid=119988027341&hvadid=585846668383&hvdev=c&hvlocphy=9069537&hvnetw=g&hvqmt=e&hvrand=585687338914204919&hvtargid=kwd-6398167069&hydadcr=5368_13199200&keywords=lightning+deals&qid=1686751432&sr=8-1-spons&sp_csd=d2lkZ2V0TmFtZT1zcF9hdGY&psc=1\n",
      "200\n"
     ]
    }
   ],
   "source": [
    "from selectorlib import Extractor\n",
    "import requests \n",
    "import json\n",
    "from time import sleep\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Create an Extractor by reading from the YAML file\n",
    "# e = Extractor.from_yaml_file('selectors.yml')\n",
    "\n",
    "def scrape(url):  \n",
    "\n",
    "    headers = {\n",
    "        'dnt': '1',\n",
    "        'upgrade-insecure-requests': '1',\n",
    "        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36',\n",
    "        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',\n",
    "        'sec-fetch-site': 'same-origin',\n",
    "        'sec-fetch-mode': 'navigate',\n",
    "        'sec-fetch-user': '?1',\n",
    "        'sec-fetch-dest': 'document',\n",
    "        'referer': 'https://www.amazon.com/',\n",
    "        'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',\n",
    "    }\n",
    "\n",
    "    print(\"Downloading %s\"%url)\n",
    "    r = requests.get(url, headers=headers)\n",
    "    print(r.status_code)\n",
    "    if r.status_code > 500:\n",
    "        if \"To discuss automated access to Amazon data please contact\" in r.text:\n",
    "            print(\"Page %s was blocked by Amazon. Please try using better proxies\\n\"%url)\n",
    "        else:\n",
    "            print(\"Page %s must have been blocked by Amazon as the status code was %d\"%(url,r.status_code))\n",
    "        return None\n",
    "    \n",
    "    soup = BeautifulSoup(r.text, \"html.parser\")\n",
    "\n",
    "    name = None\n",
    "    selector1 = \"#productTitle\"\n",
    "    el = soup.select_one(selector1)\n",
    "    if el:\n",
    "        name = el.get_text()\n",
    "    else:\n",
    "        selector2 = \"#alternateProductName\"\n",
    "        el = soup.select_one(selector2)\n",
    "        if el:\n",
    "            name = el.get_text()\n",
    "\n",
    "    price = None\n",
    "    selectors = [\n",
    "        \"span#price_inside_buybox\",\n",
    "        \"#alternative_price\",\n",
    "        \"span.a-price.aok-align-center.reinventPricePriceToPayMargin.priceToPay\"]\n",
    "\n",
    "    for selector in selectors:\n",
    "        el = soup.select_one(selector)\n",
    "        if el:\n",
    "            price = el.get_text()\n",
    "            price = split(price)\n",
    "            break\n",
    "    int_price = float((price.replace(\"$\", \"\")).replace(\" \", \"\"))\n",
    "            \n",
    "    original_price = None\n",
    "    selectors = [\n",
    "        \"td.a-span12.a-color-secondary span.a-price\",\n",
    "        \"span.a-price[data-a-strike='true']\"]  \n",
    "\n",
    "    for selector in selectors:\n",
    "        el = soup.select_one(selector)\n",
    "        if el:\n",
    "            original_price = el.get_text()\n",
    "            original_price = split(original_price)\n",
    "            break\n",
    "        \n",
    "    if original_price:\n",
    "        int_original = float((original_price.replace(\"$\", \"\")).replace(\" \", \"\"))\n",
    "        discount = str(round((1 - int_price / int_original) * 100)) + \"%\"\n",
    "    else:\n",
    "        original_price = price\n",
    "        discount = None\n",
    "\n",
    "    data = {\n",
    "       \"name\": name,\n",
    "       \"price\": price,\n",
    "       \"discount\": discount,\n",
    "       \"original price\": original_price,\n",
    "    }\n",
    "    \n",
    "    return data\n",
    "      \n",
    "with open(\"urls.txt\") as f, open(\"output.jsonl\", \"w\") as outfile:    \n",
    "    for url in f:\n",
    "        data = scrape(url.strip())  \n",
    "          \n",
    "        if data:     \n",
    "           json.dump(data, outfile)           \n",
    "           outfile.write(\"\\n\")  \n",
    "           sleep(5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}