001 - books scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-4-3fae2e29d8ab>:20: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  driver = webdriver.Chrome(PATH)\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "Created on Sun May  2 14:58:07 2021\n",
    "\n",
    "@author: ahmed eltabakh\n",
    "\"\"\"\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "\n",
    "\n",
    "# Scrape one category # Travel\n",
    "\n",
    "#Set up the path to the chrome driver\n",
    "PATH = \"C:\\chromedriver_win32_n\\chromedriver.exe\"\n",
    "driver = webdriver.Chrome(PATH)\n",
    "#parse the page source using get() function\n",
    "driver.get(\"http://books.toscrape.com/catalogue/category/books_1/index.html\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-5-00be785ed70b>:1: DeprecationWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
      "  incategory = driver.find_elements_by_class_name(\"product_pod\")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"2768557c-e68d-4b42-b182-53e5036da2d9\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"5a2ca3e4-9c2c-4a2d-a480-0b5332314113\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"35513a9e-718a-4783-a564-0c6ee8d3e7d0\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"db82b444-08e0-437d-9443-9f3fc3aeab13\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"5ef72536-a0a7-4c73-b3b9-1f79faab4dbe\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"598fc715-b396-4e0e-b2c4-020585ee6f9c\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"f826cb32-30ab-453b-9d1e-be5a88dac028\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"c668660f-d232-4520-acd7-d48a468576ad\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"d31c3b3d-e142-4d29-9d5e-3923436eee22\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"1a7b3292-12f9-4296-a540-69c94663b666\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"018b69fc-4e12-4933-b7e0-478f7cf8c060\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"4d8d5f00-63ca-44ad-855a-12a8e6f9e586\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"9186b5bc-7400-4fba-a006-70f642987e07\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"ffc03564-fd4f-41c6-abe7-f436c701e6f0\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"10d66d84-43a0-4820-b81d-4cb02ff8a424\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"83908c7f-ad5f-41ce-ab78-4754c749ac8a\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"27414bf8-ce1b-445c-96ea-8f892fc6408e\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"842dc705-be29-4066-ad3f-69c5153fb722\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"477277e4-9ee4-4fe0-b351-e1077b523597\")>,\n",
       " <selenium.webdriver.remote.webelement.WebElement (session=\"833b5aa0b2d89bab0273528ec98adcb8\", element=\"7be42767-2ccd-42ac-a24c-5658cb434d71\")>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "incategory = driver.find_elements_by_class_name(\"product_pod\")\n",
    "incategory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ahmed eltabakh\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py:340: UserWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  warnings.warn(\"find_element_by_* commands are deprecated. Please use find_element() instead\")\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'links' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-3f8b19ddb80a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m                         \u001b[0ma\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mitem\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_tag_name\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"h3\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_tag_name\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"a\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_property\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"href\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m                         \u001b[1;31m#Append the link to list links\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m                         \u001b[0mlinks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'links' is not defined"
     ]
    }
   ],
   "source": [
    "for i in range(len(incategory)):\n",
    "\t\t\titem = incategory[i]\n",
    "\t\t\t#get the href property\n",
    "\t\t\ta = item.find_element_by_tag_name(\"h3\").find_element_by_tag_name(\"a\").get_property(\"href\")\n",
    "\t\t\t#Append the link to list links\n",
    "\t\t\tlinks.append(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://books.toscrape.com/catalogue/category/books_1/page-1.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-2.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-3.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-4.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-5.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-6.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-7.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-8.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-9.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-10.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-11.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-12.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-13.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-14.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-15.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-16.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-17.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-18.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-19.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-20.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-21.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-22.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-23.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-24.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-25.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-26.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-27.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-28.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-29.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-30.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-31.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-32.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-33.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-34.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-35.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-36.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-37.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-38.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-39.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-40.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-41.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-42.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-43.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-44.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-45.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-46.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-47.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-48.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-49.html\n",
      "http://books.toscrape.com/catalogue/category/books_1/page-50.html\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "Created on Sun May  2 14:58:07 2021\n",
    "\n",
    "@author: ahmed eltabakh\n",
    "\"\"\"\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "\n",
    "\n",
    "# Scrape one category # Travel\n",
    "\n",
    "#Set up the path to the chrome driver\n",
    "PATH = \"C:\\chromedriver_win32\\chromedriver.exe\"\n",
    "driver = webdriver.Chrome(PATH)\n",
    "#parse the page source using get() function\n",
    "driver.get(\"http://books.toscrape.com/catalogue/category/books_1/index.html\")\n",
    "\n",
    "\n",
    "def StarConversion(value):\n",
    "\tif value == \"One\":\n",
    "\t\treturn 1\n",
    "\telif value == \"Two\":\n",
    "\t\treturn 2\n",
    "\telif value == \"Three\":\n",
    "\t\treturn 3\n",
    "\telif value == \"Four\":\n",
    "\t\treturn 4\n",
    "\telif value == \"Five\":\n",
    "\t\treturn 5 \n",
    "\n",
    "\n",
    "\n",
    "#next_button = driver.find_element_by_class_name(\"next\").find_element_by_tag_name(\"a\").click()\n",
    "all_details = []\n",
    "for c in range(1,51):\n",
    "\ttry:\n",
    "\t\t#get the page\n",
    "\t\tdriver.get(\"http://books.toscrape.com/catalogue/category/books_1/page-{}.html\".format(c))\n",
    "\t\tprint(\"http://books.toscrape.com/catalogue/category/books_1/page-{}.html\".format(c))\n",
    "\t\t# Lets find all books in the page\n",
    "\t\tincategory = driver.find_elements_by_class_name(\"product_pod\")\n",
    "\t\t#Generate a list of links for each and every book\n",
    "\t\tlinks = []\n",
    "\t\tfor i in range(len(incategory)):\n",
    "\t\t\titem = incategory[i]\n",
    "\t\t\t#get the href property\n",
    "\t\t\ta = item.find_element_by_tag_name(\"h3\").find_element_by_tag_name(\"a\").get_property(\"href\")\n",
    "\t\t\t#Append the link to list links\n",
    "\t\t\tlinks.append(a)\n",
    "\n",
    "\t\t# Lets loop through each link to acces the page of each book\n",
    "\t\tfor link in links:\n",
    "\t\t\t# get one book url\n",
    "\t\t\tdriver.get(url=link)\n",
    "\t\t\t# title of the book\n",
    "\t\t\ttitle = driver.find_element_by_xpath(\"//*[@id='content_inner']/article/div[1]/div[2]/h1\")\n",
    "\t\t\t# price of the book\n",
    "\t\t\tprice = driver.find_element_by_xpath(\"//*[@id='content_inner']/article/div[1]/div[2]/p[1]\")\n",
    "\t\t\t# stock - number of copies available for the book\n",
    "\t\t\tstock = driver.find_element_by_xpath(\"//*[@id='content_inner']/article/div[1]/div[2]/p[2]\")\n",
    "\t\t\t# Stock comes as string so we need to use this regex to exract digits\n",
    "\t\t\tstock = int(re.findall(\"\\d+\",stock.text)[0])\n",
    "\t\t\t# Stars - Actual stars are values of class attribute\n",
    "\t\t\tstars = driver.find_element_by_xpath(\"//*[@id='content_inner']/article/div[1]/div[2]/p[3]\").get_attribute(\"class\")\n",
    "\t\t\t# convert string to number. Stars are like One, Two, Three ... We need 1,2,3,...\n",
    "\t\t\tstars = StarConversion(stars.split()[1])\n",
    "\t\t\t# Description\n",
    "\t\t\ttry:\n",
    "\t\t\t\tdescription = driver.find_element_by_xpath(\"//*[@id='content_inner']/article/p\")\n",
    "\t\t\t\tdescription = description.text\n",
    "\t\t\texcept:\n",
    "\t\t\t\tdescription = None\n",
    "\t\t\t# UPC ID\n",
    "\t\t\tupc = driver.find_element_by_xpath(\"//*[@id='content_inner']/article/table/tbody/tr[1]/td\")\n",
    "\t\t\t# Tax imposed in the book\n",
    "\t\t\ttax = driver.find_element_by_xpath(\"//*[@id='content_inner']/article/table/tbody/tr[5]/td\")\n",
    "\t\t\t# Category of the book\n",
    "\t\t\tcategory_a =  driver.find_element_by_xpath(\"//*[@id='default']/div/div/ul/li[3]/a\")\n",
    "\n",
    "\t\t\t# Define a dictionary with details we need\n",
    "\t\t\tr = {\n",
    "\t\t\t\t\"1Title\":title.text,\n",
    "\t\t\t\t\"2Category\":category_a.text,\n",
    "\t\t\t\t\"3Stock\": stock,\n",
    "\t\t\t\t\"4Stars\": stars,\n",
    "\t\t\t\t\"5Price\":price.text,\n",
    "\t\t\t\t\"6Tax\":tax.text,\n",
    "\t\t\t\t\"7UPC\":upc.text,\n",
    "\t\t\t\t\"8Description\": description\n",
    "\t\t\t}\n",
    "\t\t\t# append r to all details\n",
    "\t\t\tall_details.append(r)\n",
    "\texcept:\n",
    "\t\t# Lets just close the browser if we run to an error\n",
    "\t\tdriver.close()\n",
    "\n",
    "# save the information into a CSV file\n",
    "df = pd.DataFrame(all_details)\n",
    "df.to_csv(\"all_pages.csv\")\n",
    "\n",
    "time.sleep(3)\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}