002 - addidas scrap.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import time\n",
    "import csv\n",
    "import random\n",
    "import math\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "#chrome deiver path\n",
    "chromepath = \"C:\\chromedriver_win32\\chromedriver.exe\"\n",
    "\n",
    "s = Service(chromepath)\n",
    "\n",
    "driver = webdriver.Chrome(service=s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-185-760922f3437e>:9: DeprecationWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
      "  cols = driver.find_elements_by_css_selector('div[class=\"col-5 tridion-col\"]')\n",
      "C:\\Users\\ahmed eltabakh\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py:358: UserWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
      "  warnings.warn(\"find_elements_by_* commands are deprecated. Please use find_elements() instead\")\n",
      "<ipython-input-185-760922f3437e>:25: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  html = driver.find_element_by_tag_name('html')\n",
      "<ipython-input-185-760922f3437e>:30: DeprecationWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
      "  str_n = driver.find_elements_by_css_selector('span[class=\"gl-body gl-body--s gl-no-margin-bottom\"]')[1].text\n",
      "<ipython-input-185-760922f3437e>:48: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  html = driver.find_element_by_tag_name('html')\n",
      "<ipython-input-185-760922f3437e>:54: DeprecationWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
      "  website_name = driver.find_elements_by_css_selector('a[class=\"gl-link\"]')[-1].get_attribute(\"href\")\n",
      "<ipython-input-185-760922f3437e>:60: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  subcategory = driver.find_element_by_css_selector('h1[class=\"heading___1_p74\"]').text\n",
      "<ipython-input-185-760922f3437e>:66: DeprecationWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
      "  products = driver.find_elements_by_css_selector('div[class=\"grid-item___3rAkS content-visibility-auto\"]')\n",
      "C:\\Users\\ahmed eltabakh\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py:483: UserWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  warnings.warn(\"find_element_by_* commands are deprecated. Please use find_element() instead\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-185-760922f3437e>:86: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  html = driver.find_element_by_tag_name('html')\n",
      "<ipython-input-185-760922f3437e>:97: DeprecationWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
      "  s = driver.find_elements_by_css_selector('h1[class=\"gl-heading gl-heading--regular gl-heading--italic name___1EbZs\"]')\n",
      "<ipython-input-185-760922f3437e>:105: DeprecationWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
      "  s = driver.find_elements_by_css_selector('div[class=\"gl-price-item notranslate\"]')\n",
      "<ipython-input-185-760922f3437e>:113: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  description = driver.find_element_by_css_selector('div[class=\"text-content___1EWJO\"]').find_element_by_tag_name('p').text\n",
      "<ipython-input-185-760922f3437e>:123: DeprecationWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
      "  sizes = driver.find_elements_by_css_selector('button[class=\"gl-label size___TqqSo\"]')\n",
      "<ipython-input-185-760922f3437e>:134: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  colors = driver.find_element_by_css_selector('h5[class=\"gl-label gl-label--s color-desktop___sl3Rb\"]').text\n",
      "<ipython-input-185-760922f3437e>:142: DeprecationWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
      "  img_divs = driver.find_elements_by_css_selector('div[class=\"view___CgbJj\"]')\n",
      "C:\\Users\\ahmed eltabakh\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py:340: UserWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  warnings.warn(\"find_element_by_* commands are deprecated. Please use find_element() instead\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48\n"
     ]
    }
   ],
   "source": [
    "description_list = []\n",
    "data_dict = {\"websitename\" : [], \"subcategory\": [], \"productname\": [], \"price\": [], \"description\": [], \"sizes\": [],\\\n",
    "                 \"color\": [], \"imgs\": []}\n",
    "\n",
    "driver.get(\"https://www.adidas.com/us\")\n",
    "time.sleep(2)\n",
    "\n",
    "# all cols # 28 col\n",
    "cols = driver.find_elements_by_css_selector('div[class=\"col-5 tridion-col\"]')\n",
    "\n",
    "# all sub categories links # 305 link\n",
    "sublinks = []\n",
    "for col in cols:\n",
    "    lis = col.find_elements_by_tag_name('li')\n",
    "    for li in lis:\n",
    "        for li in li.find_elements_by_tag_name('a'):\n",
    "            sublinks.append(li.get_attribute(\"href\"))\n",
    "\n",
    "# get only the first one link #category\n",
    "driver.get(sublinks[0])\n",
    "time.sleep(2)\n",
    "\n",
    "#scroll down\n",
    "for i in range(11):\n",
    "    html = driver.find_element_by_tag_name('html')\n",
    "    html.send_keys(Keys.PAGE_DOWN)\n",
    "    time.sleep(1)\n",
    "    \n",
    "# calculate num pages\n",
    "str_n = driver.find_elements_by_css_selector('span[class=\"gl-body gl-body--s gl-no-margin-bottom\"]')[1].text\n",
    "num_pages = int(re.findall(r'\\d+', str_n)[0])\n",
    "\n",
    "#extract pages links \n",
    "link = sublinks[0] + \"?start=\"\n",
    "first_link = sublinks[0] + \"?start=1\"\n",
    "links = [first_link]\n",
    "for i in range(1,num_pages):\n",
    "    page_link = link + str(i * 48)\n",
    "    links.append(page_link)\n",
    "\n",
    "for page_link in links:\n",
    "    # get page\n",
    "    driver.get(page_link)\n",
    "    time.sleep(2)\n",
    "    \n",
    "    #scroll down\n",
    "    for i in range(10):\n",
    "        html = driver.find_element_by_tag_name('html')\n",
    "        html.send_keys(Keys.PAGE_DOWN)\n",
    "        time.sleep(2)\n",
    "    \n",
    "    try:\n",
    "        # website_name\n",
    "        website_name = driver.find_elements_by_css_selector('a[class=\"gl-link\"]')[-1].get_attribute(\"href\")\n",
    "    except:\n",
    "        website_name = \"null\"\n",
    "        \n",
    "    try:\n",
    "        # subcategory\n",
    "        subcategory = driver.find_element_by_css_selector('h1[class=\"heading___1_p74\"]').text\n",
    "    except:\n",
    "        subcategory = \"null\"\n",
    "        \n",
    "    try:\n",
    "        #products in page\n",
    "        products = driver.find_elements_by_css_selector('div[class=\"grid-item___3rAkS content-visibility-auto\"]')\n",
    "    except:\n",
    "        products = []\n",
    "    \n",
    "    products_links = []\n",
    "    for product in products:\n",
    "        try:\n",
    "            link = product.find_element_by_css_selector('a[class=\"gl-product-card__assets-link\"]').get_attribute(\"href\")\n",
    "            products_links.append(link)\n",
    "        except:\n",
    "            pass\n",
    "        \n",
    "    print(len(products_links))\n",
    "    \n",
    "    for p_link in products_links:\n",
    "        try:\n",
    "            driver.get(p_link)\n",
    "            time.sleep(2)\n",
    "            #scroll down\n",
    "            for i in range(4):\n",
    "                html = driver.find_element_by_tag_name('html')\n",
    "                html.send_keys(Keys.PAGE_DOWN)\n",
    "                time.sleep(1)\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "        data_dict[\"websitename\"].append(website_name)\n",
    "        data_dict[\"subcategory\"].append(subcategory)\n",
    "\n",
    "        # product name\n",
    "        try:\n",
    "            s = driver.find_elements_by_css_selector('h1[class=\"gl-heading gl-heading--regular gl-heading--italic name___1EbZs\"]')\n",
    "            product_name = s[1].text\n",
    "            data_dict[\"productname\"].append(product_name)\n",
    "        except:\n",
    "            data_dict[\"productname\"].append(\"null\")\n",
    "\n",
    "        # price\n",
    "        try:\n",
    "            s = driver.find_elements_by_css_selector('div[class=\"gl-price-item notranslate\"]')\n",
    "            product_price = s[-1].text\n",
    "            data_dict[\"price\"].append(product_price)\n",
    "        except:\n",
    "            data_dict[\"price\"].append(\"null\")\n",
    "\n",
    "        # description\n",
    "        try:\n",
    "            description = driver.find_element_by_css_selector('div[class=\"text-content___1EWJO\"]').find_element_by_tag_name('p').text\n",
    "            data_dict[\"description\"].append(description)\n",
    "            description_list.append(description)\n",
    "        except:\n",
    "            data_dict[\"description\"].append(\"null\")\n",
    "            description_list.append(\"null\")\n",
    "\n",
    "        # product size\n",
    "        try:\n",
    "            product_sizes = []\n",
    "            sizes = driver.find_elements_by_css_selector('button[class=\"gl-label size___TqqSo\"]')\n",
    "            for size in sizes:\n",
    "                product_sizes.append(size.text)\n",
    "\n",
    "            product_size = \",\".join(product_sizes)\n",
    "            data_dict[\"sizes\"].append(product_size)\n",
    "        except:\n",
    "            data_dict[\"sizes\"].append(\"null\")\n",
    "\n",
    "        # product color\n",
    "        try:\n",
    "            colors = driver.find_element_by_css_selector('h5[class=\"gl-label gl-label--s color-desktop___sl3Rb\"]').text\n",
    "            data_dict[\"color\"].append(colors)\n",
    "        except:\n",
    "            colors = \"null\"\n",
    "            data_dict[\"color\"].append(colors)\n",
    "\n",
    "        # img links\n",
    "        try:\n",
    "            img_divs = driver.find_elements_by_css_selector('div[class=\"view___CgbJj\"]')\n",
    "            img_links = []\n",
    "            for img in img_divs:\n",
    "                img_link = img.find_element_by_tag_name(\"img\").get_attribute(\"src\")\n",
    "                img_links.append(img_link)\n",
    "\n",
    "            img_links_text = '|'.join(img_links)\n",
    "            data_dict[\"imgs\"].append(img_links_text)\n",
    "        except:\n",
    "            data_dict[\"imgs\"].append(\"null\")\n",
    "    \n",
    "print(data_dict) \n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(data_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dict['description']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "description_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_data2 = pd.DataFrame(data_dict)\n",
    "sample_data2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_data2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}