demo_plain_scrap.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ea3b5045-183d-418d-be30-e29943a13dab",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from langchain.text_splitter import TokenTextSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "9f5eaf30-99df-4de7-84f6-e424b693493f",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = TokenTextSplitter(\n",
    "        chunk_size=100, chunk_overlap=20\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a1c2638e-0a6b-4e55-aba6-65b9f2c7150b",
   "metadata": {},
   "outputs": [],
   "source": [
    "page1=\"\"\"<body>\n",
    "  <div>Britannica contains a lot of information.</div>\n",
    "<div>\n",
    "\n",
    "  <h1>Europe Countries</h1>\n",
    "  <div>\n",
    "    <h2>Germany\n",
    "    </h2>\n",
    "    <p>Germany, officially Federal Republic of Germany, German Deutschland or Bundesrepublik Deutschland, country of north-central Europe, traversing the continent’s main physical divisions, from the outer ranges of the Alps northward across the varied landscape of the Central German Uplands and then across the North German Plain.\n",
    "    </p>\n",
    "  </div>\n",
    "\n",
    "  <div>\n",
    "    <h2>France\n",
    "    </h2>\n",
    "    <p>France, officially French Republic, French France or République Française, country of northwestern Europe. Historically and culturally among the most important nations in the Western world, France has also played a highly significant role in international affairs, with former colonies in every corner of the globe.\n",
    "    </p>\n",
    "  </div>\n",
    "\n",
    "</div>\n",
    "</body>\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "1ad809c2-14f9-4ae0-8316-c42ddd80a872",
   "metadata": {},
   "outputs": [],
   "source": [
    "page2=\"\"\"<body>\n",
    "  <div>Britannica contains a lot of information.</div>\n",
    "  <div>\n",
    "\n",
    "  <h1>Asia Countries</h1>\n",
    "  <div>\n",
    "    <h2>China\n",
    "    </h2>\n",
    "    <p>China, Chinese (Pinyin) Zhonghua or (Wade-Giles romanization) Chung-hua, also spelled (Pinyin) Zhongguo or (Wade-Giles romanization) Chung-kuo, officially People’s Republic of China or Chinese (Pinyin) Zhonghua Renmin Gongheguo or (Wade-Giles romanization) Chung-hua Jen-min Kung-ho-kuo, country of East Asia. It is the largest of all Asian countries and has the largest population of any country in the world. \n",
    "    </p>\n",
    "  </div>\n",
    "\n",
    "  <div>\n",
    "    <h2>India\n",
    "    </h2>\n",
    "    <p>India, country that occupies the greater part of South Asia. Its capital is New Delhi, built in the 20th century just south of the historic hub of Old Delhi to serve as India’s administrative centre. Its government is a constitutional republic that represents a highly diverse population consisting of thousands of ethnic groups and likely hundreds of languages.\n",
    "    </p>\n",
    "  </div>\n",
    "  </div>\n",
    "</body>\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "094b8faf-d12e-4ac4-8cbc-0b4d17a31983",
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(page1, 'html.parser')\n",
    "docs1 = text_splitter.create_documents([soup.text])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "b848d351-6174-4968-83ad-8368c5c23862",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\nBritannica contains a lot of information.\\n\\nEurope Countries\\n\\nGermany\\n    \\nGermany, officially Federal Republic of Germany, German Deutschland or Bundesrepublik Deutschland, country of north-central Europe, traversing the continent’s main physical divisions, from the outer ranges of the Alps northward across the varied landscape of the Central German Uplands and then across the North German Plain.\\n    \\n\\n\\nFrance\\n ',\n",
       " 'plands and then across the North German Plain.\\n    \\n\\n\\nFrance\\n    \\nFrance, officially French Republic, French France or République Française, country of northwestern Europe. Historically and culturally among the most important nations in the Western world, France has also played a highly significant role in international affairs, with former colonies in every corner of the globe.\\n    \\n\\n\\n',\n",
       " '\\n    \\n\\n\\n']"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[p.page_content for p in docs1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "e0bae3fb-1c14-4407-9d31-9566fbbcc44e",
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(page2, 'html.parser')\n",
    "docs2 = text_splitter.create_documents([soup.text])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "d292eac5-34ac-409c-8581-12a9c64f084d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\nBritannica contains a lot of information.\\n\\nAsia Countries\\n\\nChina\\n    \\nChina, Chinese (Pinyin) Zhonghua or (Wade-Giles romanization) Chung-hua, also spelled (Pinyin) Zhongguo or (Wade-Giles romanization) Chung-kuo, officially People’s Republic of China or Chinese (Pinyin) Zhonghua Renmin Gongheguo',\n",
       " 's Republic of China or Chinese (Pinyin) Zhonghua Renmin Gongheguo or (Wade-Giles romanization) Chung-hua Jen-min Kung-ho-kuo, country of East Asia. It is the largest of all Asian countries and has the largest population of any country in the world. \\n    \\n\\n\\nIndia\\n    \\nIndia, country that occupies the greater part of South Asia. Its capital is New',\n",
       " '   \\nIndia, country that occupies the greater part of South Asia. Its capital is New Delhi, built in the 20th century just south of the historic hub of Old Delhi to serve as India’s administrative centre. Its government is a constitutional republic that represents a highly diverse population consisting of thousands of ethnic groups and likely hundreds of languages.\\n    \\n\\n\\n']"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[p.page_content for p in docs2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4be3d806-43ee-47de-9092-3e8933f9879b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e216ac6-76de-464c-bc35-2ebf661aa68a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}