carmella.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Selenium web driver is availible [here](https://chromedriver.storage.googleapis.com/83.0.4103.39/chromedriver_mac64.zip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import json\n",
    "import time\n",
    "\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup, element\n",
    "from bidi.algorithm import get_display\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "webdriver_path = r'/Users/evgenys/GIT/kfar_scrape/chromedriver'\n",
    "URL='https://www.carmella.co.il/product-category/%d7%99%d7%a8%d7%a7%d7%95%d7%aa/'\n",
    "DATA_PATH = '/Users/evgenys/GIT/kfar_scrape/DATA'\n",
    "CSV_FIELD_NAMES = [\n",
    "    'Product_ID',\n",
    "    'Product_Name',\n",
    "    'Description',\n",
    "    'Price',\n",
    "    'Price_Type',\n",
    "    'New_Product',\n",
    "    'Sale',\n",
    "    'Image_URL']\n",
    "no_of_pagedowns = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path to webdriver\n",
    "browser = webdriver.Chrome(webdriver_path)\n",
    "\n",
    "# URL to scrape\n",
    "browser.get(URL)\n",
    "time.sleep(1)\n",
    "\n",
    "elem = browser.find_element_by_tag_name(\"body\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We need to press down key until we get all the JS defined products"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "while no_of_pagedowns:\n",
    "    elem.send_keys(Keys.PAGE_DOWN)\n",
    "    time.sleep(1)\n",
    "    no_of_pagedowns-=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Starting HTML analyzing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(browser.page_source,\"html.parser\")\n",
    "all_products = soup.find_all('div', {\"class\": 'product_wrap'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "product = all_products[102]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "product_id = product['data-product-id']\n",
    "product_cat_name = product.find_parents('div', {\"class\": 'subcat_with_items'})[0]['name']\n",
    "product_image_url = product.find_all('div', {\"class\": 'product_img'})[0].find_all('img')[0]['data-src']\n",
    "product_block = product.find_all('div', {\"class\": 'prod_bottom'})[0]\n",
    "product_name = product_block.find_all('h3', {\"class\": 'pr_title'})[0].contents[0]\n",
    "product_price = product_block.find_all('span', {\"class\": 'pr_price'})[0].contents[0].strip()\n",
    "product_price_type = product_block.find_all('span', {\"class\": 'pr_price_kilo'})[0].contents[0].replace('/', '').strip()\n",
    "\n",
    "isSale = 0\n",
    "isNew = 0\n",
    "if len(product_block.find_all('span', {\"class\": 'pr_title_sale'})) or len(product_block.find_all('span', {\"class\": 'pr_price_old'})): \n",
    "    isSale = 1\n",
    "\n",
    "product_name_extra = ''\n",
    "if len(product_block.find_all('span', {\"class\": 'pr_title_note'})):\n",
    "    product_name_extra = product_block.find_all('span', {\"class\": 'pr_title_note'})[0].contents[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "row = {\n",
    "    'Product_ID': product_id,\n",
    "    'Product_Name': product_name,\n",
    "    'Description': product_name_extra,\n",
    "    'Price': product_price,\n",
    "    'Price_Type': product_price_type,\n",
    "    'New_Product': isNew,\n",
    "    'Sale': isSale,\n",
    "    'Image_URL': product_image_url}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Product_ID': '52921',\n",
       " 'Product_Name': 'שום שחור קלוף - Eravital',\n",
       " 'Description': '125 גרם',\n",
       " 'Price': '39.90',\n",
       " 'Price_Type': 'יח׳',\n",
       " 'New_Product': 0,\n",
       " 'Sale': 1,\n",
       " 'Image_URL': 'https://w9y3w7q3.stackpathcdn.com/wp-content/uploads/2020/02/8681644498969-300x255.jpg'}"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}