From 9e23157ace9c6072fedf1920828a272f271c86af Mon Sep 17 00:00:00 2001 From: luislague Date: Wed, 30 Oct 2024 10:14:55 +0100 Subject: [PATCH] Solution --- .DS_Store | Bin 0 -> 6148 bytes lab-langchain-evaluation.ipynb | 766 +++++++++++++++++++-------------- 2 files changed, 441 insertions(+), 325 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..4d883f2ba9822f416a839a12dd6f053eea54c746 GIT binary patch literal 6148 zcmeHK%}T>S5Z-O0O({YS3Oz1(E!Y+-7B3;z7cim+m70*E!I&*gY7V84v%Zi|;`2DO zyAg}^Dq?3~_nY6{><8H&#u)eJQI9c)F=j(UbgvE7Ofn+JF~TC7hKUTpelxMZ z4*2aBi&(&N7JU8waGJzf-tE5gTHV^(ZrBaGW#9S_vhedDpJ!e$yGH9m$|S7xAiRtx z#n{13t~;%EXXH&<~Q$-8SivhOpX+({XK5nSd+=5HRURQRKnxHA#K7h< zV9o@)wYe0~%83DD;0Fe9e-O|RU4x}YwRJ#;*Jq4dh$x`rTLMuSbPbjoAp*j6DWERp z=83^|IrxRia}Aanbvff|W*EoJTs>a6njQQ?r8Dkoq@EZc238rUYtzB={|tVawU7MO z60(Q^V&I=Kz^#!#@?cTsZ2h)8JZlBCJ7_4FSE2#}`pP8$4BSWBDyZWEb;xrKmKt#s S^s90}x(Fyjs3Qh`fq@SJAxb^~ literal 0 HcmV?d00001 diff --git a/lab-langchain-evaluation.ipynb b/lab-langchain-evaluation.ipynb index 3759203..2cbd668 100644 --- a/lab-langchain-evaluation.ipynb +++ b/lab-langchain-evaluation.ipynb @@ -30,7 +30,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 9, "id": "b7ed03ed-1322-49e3-b2a2-33e94fb592ef", "metadata": { "height": 98, @@ -42,20 +42,11 @@ "import os\n", "_ = load_dotenv(find_dotenv())\n", "\n", - "OPENAI_API_KEY = os.getenv('OPENAI_API_KEY') " + "OPENAI_API_KEY = os.getenv('OPENAI_API_KEY') \n" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "ce74685f-50e9-48cf-8e2c-fac05e8e3ffd", - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "markdown", - "id": "e7e29d2c-ba67-4cba-8ded-375fe040b9ba", "metadata": {}, "source": [ "### Example 1" @@ -71,54 +62,50 @@ }, { "cell_type": "code", - "execution_count": 57, - "id": "974acf8e-8f88-42de-88f8-40a82cb58e8b", - "metadata": { - "height": 115, - "tags": [] - }, + "execution_count": 52, + "id": "cf86df0b", + "metadata": {}, "outputs": [], "source": [ "from langchain.chains import RetrievalQA\n", - "from langchain_openai import ChatOpenAI\n", + "from langchain.chat_models import ChatOpenAI\n", "from langchain.llms import OpenAI\n", "from langchain_huggingface import HuggingFaceEmbeddings\n", "from langchain.document_loaders import CSVLoader, TextLoader\n", "from langchain.indexes import VectorstoreIndexCreator\n", "from langchain.vectorstores import DocArrayInMemorySearch\n", - "from langchain.chains import LLMChain\n" + "from langchain.chains import LLMChain\n", + "import nest_asyncio\n", + "nest_asyncio.apply()" ] }, { "cell_type": "code", - "execution_count": 7, - "id": "9ec1106d", - "metadata": { - "height": 64, - "tags": [] - }, + "execution_count": 3, + "id": "230090f0", + "metadata": {}, "outputs": [], "source": [ - "file = '../data/OutdoorClothingCatalog_1000.csv'\n", + "file = '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/OutdoorClothingCatalog_1000.csv'\n", "loader = CSVLoader(file_path=file)\n", "data = loader.load()" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 15, "id": "550eb642-c223-4d78-8f92-0f265ef78b86", "metadata": { "tags": [] }, "outputs": [], "source": [ - "# !pip install --upgrade --force-reinstall sentence-transformers" + "#!pip install --upgrade --force-reinstall sentence-transformers\n" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 4, "id": "b31c218f", "metadata": { "height": 64, @@ -129,10 +116,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "/opt/anaconda3/lib/python3.11/site-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The class `HuggingFaceEmbeddings` was deprecated in LangChain 0.2.2 and will be removed in 0.3.0. An updated version of the class exists in the langchain-huggingface package and should be used instead. To use it run `pip install -U langchain-huggingface` and import as `from langchain_huggingface import HuggingFaceEmbeddings`.\n", - " warn_deprecated(\n", - "/opt/anaconda3/lib/python3.11/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:11: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", - " from tqdm.autonotebook import tqdm, trange\n" + "/Users/zone/.local/lib/python3.12/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:13: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", + " from tqdm.autonotebook import tqdm, trange\n", + "/Users/zone/.local/lib/python3.12/site-packages/pydantic/_migration.py:283: UserWarning: `pydantic.error_wrappers:ValidationError` has been moved to `pydantic:ValidationError`.\n", + " warnings.warn(f'`{import_path}` has been moved to `{new_location}`.')\n" ] } ], @@ -145,22 +132,13 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 10, "id": "a2006054", "metadata": { "height": 183, "tags": [] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/anaconda3/lib/python3.11/site-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The class `ChatOpenAI` was deprecated in LangChain 0.0.10 and will be removed in 0.3.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import ChatOpenAI`.\n", - " warn_deprecated(\n" - ] - } - ], + "outputs": [], "source": [ "llm = ChatOpenAI(temperature = 0.0)\n", "qa = RetrievalQA.from_chain_type(\n", @@ -184,7 +162,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 11, "id": "fb04a0f9", "metadata": { "height": 30, @@ -194,10 +172,10 @@ { "data": { "text/plain": [ - "Document(page_content=\": 10\\nname: Cozy Comfort Pullover Set, Stripe\\ndescription: Perfect for lounging, this striped knit set lives up to its name. We used ultrasoft fabric and an easy design that's as comfortable at bedtime as it is when we have to make a quick run out.\\n\\nSize & Fit\\n- Pants are Favorite Fit: Sits lower on the waist.\\n- Relaxed Fit: Our most generous fit sits farthest from the body.\\n\\nFabric & Care\\n- In the softest blend of 63% polyester, 35% rayon and 2% spandex.\\n\\nAdditional Features\\n- Relaxed fit top with raglan sleeves and rounded hem.\\n- Pull-on pants have a wide elastic waistband and drawstring, side pockets and a modern slim leg.\\n\\nImported.\", metadata={'source': '../data/OutdoorClothingCatalog_1000.csv', 'row': 10})" + "Document(metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/OutdoorClothingCatalog_1000.csv', 'row': 10}, page_content=\": 10\\nname: Cozy Comfort Pullover Set, Stripe\\ndescription: Perfect for lounging, this striped knit set lives up to its name. We used ultrasoft fabric and an easy design that's as comfortable at bedtime as it is when we have to make a quick run out.\\n\\nSize & Fit\\n- Pants are Favorite Fit: Sits lower on the waist.\\n- Relaxed Fit: Our most generous fit sits farthest from the body.\\n\\nFabric & Care\\n- In the softest blend of 63% polyester, 35% rayon and 2% spandex.\\n\\nAdditional Features\\n- Relaxed fit top with raglan sleeves and rounded hem.\\n- Pull-on pants have a wide elastic waistband and drawstring, side pockets and a modern slim leg.\\n\\nImported.\")" ] }, - "execution_count": 15, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -208,7 +186,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 12, "id": "fe4a88c2", "metadata": { "height": 30, @@ -218,10 +196,10 @@ { "data": { "text/plain": [ - "Document(page_content=': 11\\nname: Ultra-Lofty 850 Stretch Down Hooded Jacket\\ndescription: This technical stretch down jacket from our DownTek collection is sure to keep you warm and comfortable with its full-stretch construction providing exceptional range of motion. With a slightly fitted style that falls at the hip and best with a midweight layer, this jacket is suitable for light activity up to 20° and moderate activity up to -30°. The soft and durable 100% polyester shell offers complete windproof protection and is insulated with warm, lofty goose down. Other features include welded baffles for a no-stitch construction and excellent stretch, an adjustable hood, an interior media port and mesh stash pocket and a hem drawcord. Machine wash and dry. Imported.', metadata={'source': '../data/OutdoorClothingCatalog_1000.csv', 'row': 11})" + "Document(metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/OutdoorClothingCatalog_1000.csv', 'row': 11}, page_content=': 11\\nname: Ultra-Lofty 850 Stretch Down Hooded Jacket\\ndescription: This technical stretch down jacket from our DownTek collection is sure to keep you warm and comfortable with its full-stretch construction providing exceptional range of motion. With a slightly fitted style that falls at the hip and best with a midweight layer, this jacket is suitable for light activity up to 20° and moderate activity up to -30°. The soft and durable 100% polyester shell offers complete windproof protection and is insulated with warm, lofty goose down. Other features include welded baffles for a no-stitch construction and excellent stretch, an adjustable hood, an interior media port and mesh stash pocket and a hem drawcord. Machine wash and dry. Imported.')" ] }, - "execution_count": 20, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -240,29 +218,27 @@ }, { "cell_type": "code", - "execution_count": 29, - "id": "106fbd91-f7bc-4d6b-b090-54b6a485ce39", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from langchain.prompts import PromptTemplate" - ] - }, - { - "cell_type": "code", - "execution_count": 55, + "execution_count": 14, "id": "0c5d5af6-36db-4421-b635-46384e677847", "metadata": { "tags": [] }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/qp/3cxfqxvd0dq7rn_xc3tvtcz00000gp/T/ipykernel_54096/1856079041.py:46: LangChainDeprecationWarning: The class `LLMChain` was deprecated in LangChain 0.1.17 and will be removed in 1.0. Use :meth:`~RunnableSequence, e.g., `prompt | llm`` instead.\n", + " llm_chain = LLMChain(\n", + "/var/folders/qp/3cxfqxvd0dq7rn_xc3tvtcz00000gp/T/ipykernel_54096/1856079041.py:56: LangChainDeprecationWarning: The method `Chain.run` was deprecated in langchain 0.1.0 and will be removed in 1.0. Use :meth:`~invoke` instead.\n", + " result = llm_chain.run({\"query\": query})\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "answer='Yes, the Cozy Comfort Pullover Set is available in black, grey, and navy blue.'\n" + "answer='Yes, it is available in multiple colors such as grey, navy, and black.'\n" ] } ], @@ -338,7 +314,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 15, "id": "d44f8376", "metadata": { "height": 64, @@ -351,7 +327,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 16, "id": "34e87816", "metadata": { "height": 47, @@ -364,7 +340,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 17, "id": "acb34772-368f-4b5e-b4bd-da9b637cc7e8", "metadata": {}, "outputs": [], @@ -374,7 +350,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 18, "id": "62abae09", "metadata": { "height": 64, @@ -385,8 +361,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "/opt/anaconda3/lib/python3.11/site-packages/langchain/chains/llm.py:368: UserWarning: The apply_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n", - " warnings.warn(\n" + "/Users/zone/.local/lib/python3.12/site-packages/langchain/chains/llm.py:369: UserWarning: The apply_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n", + " warnings.warn(\n", + "/Users/zone/.local/lib/python3.12/site-packages/langchain_community/chat_models/openai.py:173: UserWarning: Unexpected type for token usage: \n", + " warnings.warn(f\"Unexpected type for token usage: {type(new_usage)}\")\n" ] } ], @@ -398,7 +376,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 19, "id": "97ab28b5", "metadata": { "height": 30, @@ -408,11 +386,11 @@ { "data": { "text/plain": [ - "{'qa_pairs': {'query': \"According to the document, what is the approximate weight of the Women's Campside Oxfords per pair?\",\n", - " 'answer': \"The approximate weight of the Women's Campside Oxfords per pair is 1 lb. 1 oz.\"}}" + "{'qa_pairs': {'query': \"What is the approximate weight of one pair of Women's Campside Oxfords?\",\n", + " 'answer': \"The approximate weight of one pair of Women's Campside Oxfords is 1 lb.1 oz.\"}}" ] }, - "execution_count": 79, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -423,7 +401,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 20, "id": "0ebe4228", "metadata": { "height": 30, @@ -433,10 +411,10 @@ { "data": { "text/plain": [ - "Document(page_content=\": 0\\nname: Women's Campside Oxfords\\ndescription: This ultracomfortable lace-to-toe Oxford boasts a super-soft canvas, thick cushioning, and quality construction for a broken-in feel from the first time you put them on. \\n\\nSize & Fit: Order regular shoe size. For half sizes not offered, order up to next whole size. \\n\\nSpecs: Approx. weight: 1 lb.1 oz. per pair. \\n\\nConstruction: Soft canvas material for a broken-in feel and look. Comfortable EVA innersole with Cleansport NXT® antimicrobial odor control. Vintage hunt, fish and camping motif on innersole. Moderate arch contour of innersole. EVA foam midsole for cushioning and support. Chain-tread-inspired molded rubber outsole with modified chain-tread pattern. Imported. \\n\\nQuestions? Please contact us for any inquiries.\", metadata={'source': '../data/OutdoorClothingCatalog_1000.csv', 'row': 0})" + "Document(metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/OutdoorClothingCatalog_1000.csv', 'row': 0}, page_content=\": 0\\nname: Women's Campside Oxfords\\ndescription: This ultracomfortable lace-to-toe Oxford boasts a super-soft canvas, thick cushioning, and quality construction for a broken-in feel from the first time you put them on. \\n\\nSize & Fit: Order regular shoe size. For half sizes not offered, order up to next whole size. \\n\\nSpecs: Approx. weight: 1 lb.1 oz. per pair. \\n\\nConstruction: Soft canvas material for a broken-in feel and look. Comfortable EVA innersole with Cleansport NXT® antimicrobial odor control. Vintage hunt, fish and camping motif on innersole. Moderate arch contour of innersole. EVA foam midsole for cushioning and support. Chain-tread-inspired molded rubber outsole with modified chain-tread pattern. Imported. \\n\\nQuestions? Please contact us for any inquiries.\")" ] }, - "execution_count": 34, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -447,7 +425,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 21, "id": "7693fe86-feeb-4d73-b400-e66e79315274", "metadata": { "tags": [] @@ -456,19 +434,19 @@ { "data": { "text/plain": [ - "[{'query': \"According to the document, what is the approximate weight of the Women's Campside Oxfords per pair?\",\n", - " 'answer': \"The approximate weight of the Women's Campside Oxfords per pair is 1 lb. 1 oz.\"},\n", + "[{'query': \"What is the approximate weight of one pair of Women's Campside Oxfords?\",\n", + " 'answer': \"The approximate weight of one pair of Women's Campside Oxfords is 1 lb.1 oz.\"},\n", " {'query': 'What are the dimensions of the small and medium sizes of the Recycled Waterhog Dog Mat, Chevron Weave?',\n", - " 'answer': 'The small size of the Recycled Waterhog Dog Mat, Chevron Weave has dimensions of 18\" x 28\", while the medium size has dimensions of 22.5\" x 34.5\".'},\n", - " {'query': \"What are some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece as described in the document?\",\n", - " 'answer': \"Some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece include bright colors, ruffles, exclusive whimsical prints, four-way-stretch and chlorine-resistant fabric, UPF 50+ rated fabric for sun protection, crossover no-slip straps, fully lined bottom for secure fit and maximum coverage. It is recommended to machine wash and line dry for best results.\"},\n", - " {'query': 'What is the fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts top?',\n", - " 'answer': 'The body of the tankini top is made of 82% recycled nylon and 18% Lycra® spandex, while the lining is made of 90% recycled nylon and 10% Lycra® spandex.'},\n", - " {'query': 'What technology does the EcoFlex 3L Storm Pants feature that makes them more breathable than previous versions?',\n", + " 'answer': 'The small size of the mat has dimensions of 18\" x 28\" and the medium size has dimensions of 22.5\" x 34.5\".'},\n", + " {'query': \"What are some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece based on the document?\",\n", + " 'answer': 'Some key features of the swimsuit include bright colors, ruffles, exclusive whimsical prints, four-way-stretch and chlorine-resistant fabric, UPF 50+ rated fabric for sun protection, crossover no-slip straps, fully lined bottom for secure fit and coverage, and the recommendation to machine wash and line dry for best results.'},\n", + " {'query': 'What is the fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts?',\n", + " 'answer': 'The Refresh Swimwear V-Neck Tankini Contrasts is made of 82% recycled nylon and 18% Lycra® spandex for the body, and lined with 90% recycled nylon and 10% Lycra® spandex.'},\n", + " {'query': 'What technology makes the EcoFlex 3L Storm Pants more breathable and comfortable?',\n", " 'answer': 'The EcoFlex 3L Storm Pants feature TEK O2 technology, which offers the most breathability ever tested by the company.'}]" ] }, - "execution_count": 65, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -488,7 +466,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 22, "id": "ada2a3fc", "metadata": { "height": 30, @@ -502,7 +480,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 23, "id": "2184b9d7-22ab-43a5-9ba5-b27fef024874", "metadata": { "tags": [] @@ -515,7 +493,7 @@ " 'answer': 'Yes'}" ] }, - "execution_count": 81, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -526,7 +504,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 24, "id": "9cdf5cf5", "metadata": { "height": 30, @@ -551,7 +529,7 @@ " 'result': 'Yes, the Cozy Comfort Pullover Set has side pockets.'}" ] }, - "execution_count": 73, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -570,7 +548,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 25, "id": "fcaf622e", "metadata": { "height": 47, @@ -584,7 +562,7 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 26, "id": "8a142638", "metadata": { "height": 30, @@ -612,12 +590,12 @@ " \"System: Use the following pieces of context to answer the user's question. \\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\\n----------------\\n: 73\\nname: Cozy Cuddles Knit Pullover Set\\ndescription: Perfect for lounging, this knit set lives up to its name. We used ultrasoft fabric and an easy design that's as comfortable at bedtime as it is when we have to make a quick run out. \\n\\nSize & Fit \\nPants are Favorite Fit: Sits lower on the waist. \\nRelaxed Fit: Our most generous fit sits farthest from the body. \\n\\nFabric & Care \\nIn the softest blend of 63% polyester, 35% rayon and 2% spandex.\\n\\nAdditional Features \\nRelaxed fit top with raglan sleeves and rounded hem. \\nPull-on pants have a wide elastic waistband and drawstring, side pockets and a modern slim leg. \\nImported.<<<<>>>>>Berber fleece-lined pockets keep hands warm. One pocket has a hidden security pocket tucked inside. Imported.<<<<>>>>>: 10\\nname: Cozy Comfort Pullover Set, Stripe\\ndescription: Perfect for lounging, this striped knit set lives up to its name. We used ultrasoft fabric and an easy design that's as comfortable at bedtime as it is when we have to make a quick run out.\\n\\nSize & Fit\\n- Pants are Favorite Fit: Sits lower on the waist.\\n- Relaxed Fit: Our most generous fit sits farthest from the body.\\n\\nFabric & Care\\n- In the softest blend of 63% polyester, 35% rayon and 2% spandex.\\n\\nAdditional Features\\n- Relaxed fit top with raglan sleeves and rounded hem.\\n- Pull-on pants have a wide elastic waistband and drawstring, side pockets and a modern slim leg.\\n\\nImported.<<<<>>>>>Additional Features: Bonded construction insulates for extra warmth and won't stretch out of shape. Classic shirt jac styling, with collar and modified hem for extra coverage. Full snap placket for easy on/off. Chest pockets with snap closures. Pencil pocket on chest. Underarm gussets enhance mobility. Berber fleece-lined pockets keep hands warm. One pocket has a hidden security pocket tucked inside. Imported\\nHuman: Do the Cozy Comfort Pullover Set have side pockets?\"\n", " ]\n", "}\n", - "\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[chain:RetrievalQA > chain:StuffDocumentsChain > chain:LLMChain > llm:ChatOpenAI] [1.27s] Exiting LLM run with output:\n", + "\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[chain:RetrievalQA > chain:StuffDocumentsChain > chain:LLMChain > llm:ChatOpenAI] [765ms] Exiting LLM run with output:\n", "\u001b[0m{\n", " \"generations\": [\n", " [\n", " {\n", - " \"text\": \"Yes, the Cozy Comfort Pullover Set has side pockets.\",\n", + " \"text\": \"Yes, the Cozy Comfort Pullover Set does have side pockets.\",\n", " \"generation_info\": {\n", " \"finish_reason\": \"stop\",\n", " \"logprobs\": null\n", @@ -633,12 +611,20 @@ " \"AIMessage\"\n", " ],\n", " \"kwargs\": {\n", - " \"content\": \"Yes, the Cozy Comfort Pullover Set has side pockets.\",\n", + " \"content\": \"Yes, the Cozy Comfort Pullover Set does have side pockets.\",\n", " \"response_metadata\": {\n", " \"token_usage\": {\n", - " \"completion_tokens\": 13,\n", + " \"completion_tokens\": 14,\n", " \"prompt_tokens\": 504,\n", - " \"total_tokens\": 517\n", + " \"total_tokens\": 518,\n", + " \"completion_tokens_details\": {\n", + " \"audio_tokens\": null,\n", + " \"reasoning_tokens\": 0\n", + " },\n", + " \"prompt_tokens_details\": {\n", + " \"audio_tokens\": null,\n", + " \"cached_tokens\": 0\n", + " }\n", " },\n", " \"model_name\": \"gpt-3.5-turbo\",\n", " \"system_fingerprint\": null,\n", @@ -646,7 +632,7 @@ " \"logprobs\": null\n", " },\n", " \"type\": \"ai\",\n", - " \"id\": \"run-a161a758-1627-4ee4-a205-3a12f334bd82-0\",\n", + " \"id\": \"run-a6e73fc3-136f-4129-833d-085c7dcd9c93-0\",\n", " \"tool_calls\": [],\n", " \"invalid_tool_calls\": []\n", " }\n", @@ -656,26 +642,35 @@ " ],\n", " \"llm_output\": {\n", " \"token_usage\": {\n", - " \"completion_tokens\": 13,\n", + " \"completion_tokens\": 14,\n", " \"prompt_tokens\": 504,\n", - " \"total_tokens\": 517\n", + " \"total_tokens\": 518,\n", + " \"completion_tokens_details\": {\n", + " \"audio_tokens\": null,\n", + " \"reasoning_tokens\": 0\n", + " },\n", + " \"prompt_tokens_details\": {\n", + " \"audio_tokens\": null,\n", + " \"cached_tokens\": 0\n", + " }\n", " },\n", " \"model_name\": \"gpt-3.5-turbo\",\n", " \"system_fingerprint\": null\n", " },\n", - " \"run\": null\n", + " \"run\": null,\n", + " \"type\": \"LLMResult\"\n", "}\n", - "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[chain:RetrievalQA > chain:StuffDocumentsChain > chain:LLMChain] [1.27s] Exiting Chain run with output:\n", + "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[chain:RetrievalQA > chain:StuffDocumentsChain > chain:LLMChain] [772ms] Exiting Chain run with output:\n", "\u001b[0m{\n", - " \"text\": \"Yes, the Cozy Comfort Pullover Set has side pockets.\"\n", + " \"text\": \"Yes, the Cozy Comfort Pullover Set does have side pockets.\"\n", "}\n", - "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[chain:RetrievalQA > chain:StuffDocumentsChain] [1.27s] Exiting Chain run with output:\n", + "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[chain:RetrievalQA > chain:StuffDocumentsChain] [773ms] Exiting Chain run with output:\n", "\u001b[0m{\n", - " \"output_text\": \"Yes, the Cozy Comfort Pullover Set has side pockets.\"\n", + " \"output_text\": \"Yes, the Cozy Comfort Pullover Set does have side pockets.\"\n", "}\n", - "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[chain:RetrievalQA] [1.30s] Exiting Chain run with output:\n", + "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[chain:RetrievalQA] [1.10s] Exiting Chain run with output:\n", "\u001b[0m{\n", - " \"result\": \"Yes, the Cozy Comfort Pullover Set has side pockets.\"\n", + " \"result\": \"Yes, the Cozy Comfort Pullover Set does have side pockets.\"\n", "}\n" ] }, @@ -683,10 +678,10 @@ "data": { "text/plain": [ "{'query': 'Do the Cozy Comfort Pullover Set have side pockets?',\n", - " 'result': 'Yes, the Cozy Comfort Pullover Set has side pockets.'}" + " 'result': 'Yes, the Cozy Comfort Pullover Set does have side pockets.'}" ] }, - "execution_count": 86, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -697,7 +692,7 @@ }, { "cell_type": "code", - "execution_count": 88, + "execution_count": 27, "id": "b3d6bef0", "metadata": { "height": 47, @@ -719,7 +714,7 @@ }, { "cell_type": "code", - "execution_count": 146, + "execution_count": 28, "id": "a54769b0-3daf-4cac-b259-89a10dd9b5a2", "metadata": { "tags": [] @@ -731,7 +726,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 29, "id": "8ea95385-1b4c-440a-9fea-8500b4cc2154", "metadata": { "tags": [] @@ -744,19 +739,29 @@ " 'answer': 'Yes'},\n", " {'query': 'What collection is the Ultra-Lofty 850 Stretch Down Hooded Jacket from?',\n", " 'answer': 'The DownTek collection'},\n", - " {'query': \"According to the document, what is the approximate weight of the Women's Campside Oxfords per pair?\",\n", - " 'answer': \"The approximate weight of the Women's Campside Oxfords per pair is 1 lb. 1 oz.\"},\n", + " {'query': \"What is the approximate weight of one pair of Women's Campside Oxfords?\",\n", + " 'answer': \"The approximate weight of one pair of Women's Campside Oxfords is 1 lb.1 oz.\"},\n", + " {'query': 'What are the dimensions of the small and medium sizes of the Recycled Waterhog Dog Mat, Chevron Weave?',\n", + " 'answer': 'The small size of the mat has dimensions of 18\" x 28\" and the medium size has dimensions of 22.5\" x 34.5\".'},\n", + " {'query': \"What are some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece based on the document?\",\n", + " 'answer': 'Some key features of the swimsuit include bright colors, ruffles, exclusive whimsical prints, four-way-stretch and chlorine-resistant fabric, UPF 50+ rated fabric for sun protection, crossover no-slip straps, fully lined bottom for secure fit and coverage, and the recommendation to machine wash and line dry for best results.'},\n", + " {'query': 'What is the fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts?',\n", + " 'answer': 'The Refresh Swimwear V-Neck Tankini Contrasts is made of 82% recycled nylon and 18% Lycra® spandex for the body, and lined with 90% recycled nylon and 10% Lycra® spandex.'},\n", + " {'query': 'What technology makes the EcoFlex 3L Storm Pants more breathable and comfortable?',\n", + " 'answer': 'The EcoFlex 3L Storm Pants feature TEK O2 technology, which offers the most breathability ever tested by the company.'},\n", + " {'query': \"What is the approximate weight of one pair of Women's Campside Oxfords?\",\n", + " 'answer': \"The approximate weight of one pair of Women's Campside Oxfords is 1 lb.1 oz.\"},\n", " {'query': 'What are the dimensions of the small and medium sizes of the Recycled Waterhog Dog Mat, Chevron Weave?',\n", - " 'answer': 'The small size of the Recycled Waterhog Dog Mat, Chevron Weave has dimensions of 18\" x 28\", while the medium size has dimensions of 22.5\" x 34.5\".'},\n", - " {'query': \"What are some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece as described in the document?\",\n", - " 'answer': \"Some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece include bright colors, ruffles, exclusive whimsical prints, four-way-stretch and chlorine-resistant fabric, UPF 50+ rated fabric for sun protection, crossover no-slip straps, fully lined bottom for secure fit and maximum coverage. It is recommended to machine wash and line dry for best results.\"},\n", - " {'query': 'What is the fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts top?',\n", - " 'answer': 'The body of the tankini top is made of 82% recycled nylon and 18% Lycra® spandex, while the lining is made of 90% recycled nylon and 10% Lycra® spandex.'},\n", - " {'query': 'What technology does the EcoFlex 3L Storm Pants feature that makes them more breathable than previous versions?',\n", + " 'answer': 'The small size of the mat has dimensions of 18\" x 28\" and the medium size has dimensions of 22.5\" x 34.5\".'},\n", + " {'query': \"What are some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece based on the document?\",\n", + " 'answer': 'Some key features of the swimsuit include bright colors, ruffles, exclusive whimsical prints, four-way-stretch and chlorine-resistant fabric, UPF 50+ rated fabric for sun protection, crossover no-slip straps, fully lined bottom for secure fit and coverage, and the recommendation to machine wash and line dry for best results.'},\n", + " {'query': 'What is the fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts?',\n", + " 'answer': 'The Refresh Swimwear V-Neck Tankini Contrasts is made of 82% recycled nylon and 18% Lycra® spandex for the body, and lined with 90% recycled nylon and 10% Lycra® spandex.'},\n", + " {'query': 'What technology makes the EcoFlex 3L Storm Pants more breathable and comfortable?',\n", " 'answer': 'The EcoFlex 3L Storm Pants feature TEK O2 technology, which offers the most breathability ever tested by the company.'}]" ] }, - "execution_count": 90, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -767,7 +772,7 @@ }, { "cell_type": "code", - "execution_count": 92, + "execution_count": 30, "id": "a4dca05a", "metadata": { "height": 30, @@ -778,6 +783,12 @@ "name": "stdout", "output_type": "stream", "text": [ + "\n", + "\n", + "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n", + "\n", + "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n", + "\n", "\n", "\n", "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n", @@ -800,6 +811,25 @@ "\n", "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n", "\n", + "\n", + "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", "\u001b[1m> Finished chain.\u001b[0m\n", "\n", "\u001b[1m> Finished chain.\u001b[0m\n", @@ -822,7 +852,7 @@ }, { "cell_type": "code", - "execution_count": 94, + "execution_count": 31, "id": "ae7e8b4e-4468-4048-8544-c9936704ea93", "metadata": { "tags": [] @@ -837,24 +867,39 @@ " {'query': 'What collection is the Ultra-Lofty 850 Stretch Down Hooded Jacket from?',\n", " 'answer': 'The DownTek collection',\n", " 'result': 'The Ultra-Lofty 850 Stretch Down Hooded Jacket is from the DownTek collection.'},\n", - " {'query': \"According to the document, what is the approximate weight of the Women's Campside Oxfords per pair?\",\n", - " 'answer': \"The approximate weight of the Women's Campside Oxfords per pair is 1 lb. 1 oz.\",\n", - " 'result': \"The approximate weight of the Women's Campside Oxfords per pair is 1 lb. 1 oz.\"},\n", + " {'query': \"What is the approximate weight of one pair of Women's Campside Oxfords?\",\n", + " 'answer': \"The approximate weight of one pair of Women's Campside Oxfords is 1 lb.1 oz.\",\n", + " 'result': \"I don't have the specific information about the weight of the Women's Campside Oxfords.\"},\n", " {'query': 'What are the dimensions of the small and medium sizes of the Recycled Waterhog Dog Mat, Chevron Weave?',\n", - " 'answer': 'The small size of the Recycled Waterhog Dog Mat, Chevron Weave has dimensions of 18\" x 28\", while the medium size has dimensions of 22.5\" x 34.5\".',\n", + " 'answer': 'The small size of the mat has dimensions of 18\" x 28\" and the medium size has dimensions of 22.5\" x 34.5\".',\n", " 'result': 'The dimensions of the small size of the Recycled Waterhog Dog Mat, Chevron Weave are 18\" x 28\", and the dimensions of the medium size are 22.5\" x 34.5\".'},\n", - " {'query': \"What are some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece as described in the document?\",\n", - " 'answer': \"Some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece include bright colors, ruffles, exclusive whimsical prints, four-way-stretch and chlorine-resistant fabric, UPF 50+ rated fabric for sun protection, crossover no-slip straps, fully lined bottom for secure fit and maximum coverage. It is recommended to machine wash and line dry for best results.\",\n", + " {'query': \"What are some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece based on the document?\",\n", + " 'answer': 'Some key features of the swimsuit include bright colors, ruffles, exclusive whimsical prints, four-way-stretch and chlorine-resistant fabric, UPF 50+ rated fabric for sun protection, crossover no-slip straps, fully lined bottom for secure fit and coverage, and the recommendation to machine wash and line dry for best results.',\n", " 'result': \"Some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece are:\\n- Bright colors, ruffles, and exclusive whimsical prints\\n- Four-way-stretch and chlorine-resistant fabric\\n- UPF 50+ rated fabric for high sun protection\\n- Crossover no-slip straps and fully lined bottom for a secure fit and coverage\\n- Machine washable and line dry for best results\"},\n", - " {'query': 'What is the fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts top?',\n", - " 'answer': 'The body of the tankini top is made of 82% recycled nylon and 18% Lycra® spandex, while the lining is made of 90% recycled nylon and 10% Lycra® spandex.',\n", - " 'result': 'The fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts top is as follows:\\n- Body: 82% recycled nylon, 18% Lycra® spandex\\n- Lining: 90% recycled nylon, 10% Lycra® spandex'},\n", - " {'query': 'What technology does the EcoFlex 3L Storm Pants feature that makes them more breathable than previous versions?',\n", + " {'query': 'What is the fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts?',\n", + " 'answer': 'The Refresh Swimwear V-Neck Tankini Contrasts is made of 82% recycled nylon and 18% Lycra® spandex for the body, and lined with 90% recycled nylon and 10% Lycra® spandex.',\n", + " 'result': 'The fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts is as follows:\\n- Body: 82% recycled nylon, 18% Lycra® spandex\\n- Lining: 90% recycled nylon, 10% Lycra® spandex'},\n", + " {'query': 'What technology makes the EcoFlex 3L Storm Pants more breathable and comfortable?',\n", " 'answer': 'The EcoFlex 3L Storm Pants feature TEK O2 technology, which offers the most breathability ever tested by the company.',\n", - " 'result': 'The EcoFlex 3L Storm Pants feature TEK O2 technology, which offers the most breathability tested by the brand.'}]" + " 'result': 'The EcoFlex 3L Storm Pants are made with TEK O2 technology, which is a state-of-the-art air-permeable technology that offers the most breathability tested. This technology helps keep the wearer dry and comfortable during various outdoor activities and weather conditions.'},\n", + " {'query': \"What is the approximate weight of one pair of Women's Campside Oxfords?\",\n", + " 'answer': \"The approximate weight of one pair of Women's Campside Oxfords is 1 lb.1 oz.\",\n", + " 'result': \"I don't have the specific information about the weight of the Women's Campside Oxfords.\"},\n", + " {'query': 'What are the dimensions of the small and medium sizes of the Recycled Waterhog Dog Mat, Chevron Weave?',\n", + " 'answer': 'The small size of the mat has dimensions of 18\" x 28\" and the medium size has dimensions of 22.5\" x 34.5\".',\n", + " 'result': 'The dimensions of the small size of the Recycled Waterhog Dog Mat, Chevron Weave are 18\" x 28\", and the dimensions of the medium size are 22.5\" x 34.5\".'},\n", + " {'query': \"What are some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece based on the document?\",\n", + " 'answer': 'Some key features of the swimsuit include bright colors, ruffles, exclusive whimsical prints, four-way-stretch and chlorine-resistant fabric, UPF 50+ rated fabric for sun protection, crossover no-slip straps, fully lined bottom for secure fit and coverage, and the recommendation to machine wash and line dry for best results.',\n", + " 'result': \"Some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece are:\\n- Bright colors, ruffles, and exclusive whimsical prints\\n- Four-way-stretch and chlorine-resistant fabric\\n- UPF 50+ rated fabric for high sun protection\\n- Crossover no-slip straps and fully lined bottom for a secure fit and coverage\\n- Machine washable and line dry for best results\"},\n", + " {'query': 'What is the fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts?',\n", + " 'answer': 'The Refresh Swimwear V-Neck Tankini Contrasts is made of 82% recycled nylon and 18% Lycra® spandex for the body, and lined with 90% recycled nylon and 10% Lycra® spandex.',\n", + " 'result': 'The fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts is as follows:\\n- Body: 82% recycled nylon, 18% Lycra® spandex\\n- Lining: 90% recycled nylon, 10% Lycra® spandex'},\n", + " {'query': 'What technology makes the EcoFlex 3L Storm Pants more breathable and comfortable?',\n", + " 'answer': 'The EcoFlex 3L Storm Pants feature TEK O2 technology, which offers the most breathability ever tested by the company.',\n", + " 'result': 'The EcoFlex 3L Storm Pants use TEK O2 technology to make them more breathable and comfortable.'}]" ] }, - "execution_count": 94, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -865,7 +910,7 @@ }, { "cell_type": "code", - "execution_count": 96, + "execution_count": 32, "id": "6012a3e0", "metadata": { "height": 30, @@ -878,7 +923,7 @@ }, { "cell_type": "code", - "execution_count": 98, + "execution_count": 33, "id": "724b1c0b", "metadata": { "height": 47, @@ -892,20 +937,29 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 34, "id": "8b46ae55", "metadata": { "height": 47, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/zone/.local/lib/python3.12/site-packages/langchain_community/chat_models/openai.py:173: UserWarning: Unexpected type for token usage: \n", + " warnings.warn(f\"Unexpected type for token usage: {type(new_usage)}\")\n" + ] + } + ], "source": [ "graded_outputs = eval_chain.evaluate(examples, predictions)" ] }, { "cell_type": "code", - "execution_count": 102, + "execution_count": 35, "id": "dc42eb35-c2d7-4581-8004-d315ade63eef", "metadata": { "tags": [] @@ -920,10 +974,15 @@ " {'results': 'CORRECT'},\n", " {'results': 'CORRECT'},\n", " {'results': 'CORRECT'},\n", + " {'results': 'CORRECT'},\n", + " {'results': 'CORRECT'},\n", + " {'results': 'CORRECT'},\n", + " {'results': 'CORRECT'},\n", + " {'results': 'CORRECT'},\n", " {'results': 'CORRECT'}]" ] }, - "execution_count": 102, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -934,7 +993,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": 38, "id": "3437cfbe", "metadata": { "height": 149, @@ -956,18 +1015,18 @@ "Predicted Answer: The Ultra-Lofty 850 Stretch Down Hooded Jacket is from the DownTek collection.\n", "\n", "Example 2:\n", - "Question: According to the document, what is the approximate weight of the Women's Campside Oxfords per pair?\n", - "Real Answer: The approximate weight of the Women's Campside Oxfords per pair is 1 lb. 1 oz.\n", - "Predicted Answer: The approximate weight of the Women's Campside Oxfords per pair is 1 lb. 1 oz.\n", + "Question: What is the approximate weight of one pair of Women's Campside Oxfords?\n", + "Real Answer: The approximate weight of one pair of Women's Campside Oxfords is 1 lb.1 oz.\n", + "Predicted Answer: I don't have the specific information about the weight of the Women's Campside Oxfords.\n", "\n", "Example 3:\n", "Question: What are the dimensions of the small and medium sizes of the Recycled Waterhog Dog Mat, Chevron Weave?\n", - "Real Answer: The small size of the Recycled Waterhog Dog Mat, Chevron Weave has dimensions of 18\" x 28\", while the medium size has dimensions of 22.5\" x 34.5\".\n", + "Real Answer: The small size of the mat has dimensions of 18\" x 28\" and the medium size has dimensions of 22.5\" x 34.5\".\n", "Predicted Answer: The dimensions of the small size of the Recycled Waterhog Dog Mat, Chevron Weave are 18\" x 28\", and the dimensions of the medium size are 22.5\" x 34.5\".\n", "\n", "Example 4:\n", - "Question: What are some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece as described in the document?\n", - "Real Answer: Some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece include bright colors, ruffles, exclusive whimsical prints, four-way-stretch and chlorine-resistant fabric, UPF 50+ rated fabric for sun protection, crossover no-slip straps, fully lined bottom for secure fit and maximum coverage. It is recommended to machine wash and line dry for best results.\n", + "Question: What are some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece based on the document?\n", + "Real Answer: Some key features of the swimsuit include bright colors, ruffles, exclusive whimsical prints, four-way-stretch and chlorine-resistant fabric, UPF 50+ rated fabric for sun protection, crossover no-slip straps, fully lined bottom for secure fit and coverage, and the recommendation to machine wash and line dry for best results.\n", "Predicted Answer: Some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece are:\n", "- Bright colors, ruffles, and exclusive whimsical prints\n", "- Four-way-stretch and chlorine-resistant fabric\n", @@ -976,16 +1035,48 @@ "- Machine washable and line dry for best results\n", "\n", "Example 5:\n", - "Question: What is the fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts top?\n", - "Real Answer: The body of the tankini top is made of 82% recycled nylon and 18% Lycra® spandex, while the lining is made of 90% recycled nylon and 10% Lycra® spandex.\n", - "Predicted Answer: The fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts top is as follows:\n", + "Question: What is the fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts?\n", + "Real Answer: The Refresh Swimwear V-Neck Tankini Contrasts is made of 82% recycled nylon and 18% Lycra® spandex for the body, and lined with 90% recycled nylon and 10% Lycra® spandex.\n", + "Predicted Answer: The fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts is as follows:\n", "- Body: 82% recycled nylon, 18% Lycra® spandex\n", "- Lining: 90% recycled nylon, 10% Lycra® spandex\n", "\n", "Example 6:\n", - "Question: What technology does the EcoFlex 3L Storm Pants feature that makes them more breathable than previous versions?\n", + "Question: What technology makes the EcoFlex 3L Storm Pants more breathable and comfortable?\n", "Real Answer: The EcoFlex 3L Storm Pants feature TEK O2 technology, which offers the most breathability ever tested by the company.\n", - "Predicted Answer: The EcoFlex 3L Storm Pants feature TEK O2 technology, which offers the most breathability tested by the brand.\n", + "Predicted Answer: The EcoFlex 3L Storm Pants are made with TEK O2 technology, which is a state-of-the-art air-permeable technology that offers the most breathability tested. This technology helps keep the wearer dry and comfortable during various outdoor activities and weather conditions.\n", + "\n", + "Example 7:\n", + "Question: What is the approximate weight of one pair of Women's Campside Oxfords?\n", + "Real Answer: The approximate weight of one pair of Women's Campside Oxfords is 1 lb.1 oz.\n", + "Predicted Answer: I don't have the specific information about the weight of the Women's Campside Oxfords.\n", + "\n", + "Example 8:\n", + "Question: What are the dimensions of the small and medium sizes of the Recycled Waterhog Dog Mat, Chevron Weave?\n", + "Real Answer: The small size of the mat has dimensions of 18\" x 28\" and the medium size has dimensions of 22.5\" x 34.5\".\n", + "Predicted Answer: The dimensions of the small size of the Recycled Waterhog Dog Mat, Chevron Weave are 18\" x 28\", and the dimensions of the medium size are 22.5\" x 34.5\".\n", + "\n", + "Example 9:\n", + "Question: What are some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece based on the document?\n", + "Real Answer: Some key features of the swimsuit include bright colors, ruffles, exclusive whimsical prints, four-way-stretch and chlorine-resistant fabric, UPF 50+ rated fabric for sun protection, crossover no-slip straps, fully lined bottom for secure fit and coverage, and the recommendation to machine wash and line dry for best results.\n", + "Predicted Answer: Some key features of the Infant and Toddler Girls' Coastal Chill Swimsuit, Two-Piece are:\n", + "- Bright colors, ruffles, and exclusive whimsical prints\n", + "- Four-way-stretch and chlorine-resistant fabric\n", + "- UPF 50+ rated fabric for high sun protection\n", + "- Crossover no-slip straps and fully lined bottom for a secure fit and coverage\n", + "- Machine washable and line dry for best results\n", + "\n", + "Example 10:\n", + "Question: What is the fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts?\n", + "Real Answer: The Refresh Swimwear V-Neck Tankini Contrasts is made of 82% recycled nylon and 18% Lycra® spandex for the body, and lined with 90% recycled nylon and 10% Lycra® spandex.\n", + "Predicted Answer: The fabric composition of the Refresh Swimwear V-Neck Tankini Contrasts is as follows:\n", + "- Body: 82% recycled nylon, 18% Lycra® spandex\n", + "- Lining: 90% recycled nylon, 10% Lycra® spandex\n", + "\n", + "Example 11:\n", + "Question: What technology makes the EcoFlex 3L Storm Pants more breathable and comfortable?\n", + "Real Answer: The EcoFlex 3L Storm Pants feature TEK O2 technology, which offers the most breathability ever tested by the company.\n", + "Predicted Answer: The EcoFlex 3L Storm Pants use TEK O2 technology to make them more breathable and comfortable.\n", "\n" ] } @@ -996,7 +1087,7 @@ " print(\"Question: \" + predictions[i]['query'])\n", " print(\"Real Answer: \" + predictions[i]['answer'])\n", " print(\"Predicted Answer: \" + predictions[i]['result'])\n", - " # print(\"Predicted Grade: \" + graded_outputs[i]['text'])\n", + " #print(\"Predicted Grade: \" + graded_outputs[i]['text'])\n", " print()" ] }, @@ -1011,7 +1102,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 39, "id": "a5ef0493-34ff-4801-b405-69c76ce86c38", "metadata": { "tags": [] @@ -1021,14 +1112,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "/opt/anaconda3/lib/python3.11/site-packages/langchain/indexes/vectorstore.py:129: UserWarning: Using InMemoryVectorStore as the default vectorstore.This memory store won't persist data. You should explicitlyspecify a vectorstore when using VectorstoreIndexCreator\n", + "/Users/zone/.local/lib/python3.12/site-packages/langchain/indexes/vectorstore.py:128: UserWarning: Using InMemoryVectorStore as the default vectorstore.This memory store won't persist data. You should explicitlyspecify a vectorstore when using VectorstoreIndexCreator\n", " warnings.warn(\n" ] } ], "source": [ "from langchain_huggingface import HuggingFaceEmbeddings\n", - "loader = TextLoader(\"../data/nyc_text.txt\")\n", + "loader = TextLoader(\"/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/nyc_text.txt\")\n", "index = VectorstoreIndexCreator(embedding=HuggingFaceEmbeddings(model_name=\"all-MiniLM-L6-v2\", model_kwargs = {'device': 'mps'})).from_loaders([loader])\n", "\n", "\n", @@ -1042,29 +1133,19 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 40, "id": "f0449cae-de25-4ef6-ae64-78ccf5e06a5a", "metadata": { "tags": [] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" - ] - }, { "data": { "text/plain": [ "'New York City was originally named New Amsterdam by Dutch colonists in 1626. When the city came under British control in 1664, it was renamed New York after King Charles II of England granted the lands to his brother, the Duke of York. The city has been continuously named New York since November 1674.'" ] }, - "execution_count": 26, + "execution_count": 40, "metadata": {}, "output_type": "execute_result" } @@ -1079,7 +1160,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 41, "id": "9e846b3d-f79f-46eb-8075-c816268c0500", "metadata": { "tags": [] @@ -1090,13 +1171,13 @@ "text/plain": [ "{'query': 'How did New York City get its name?',\n", " 'result': 'New York City was originally named New Amsterdam by Dutch colonists in 1626. When the city came under British control in 1664, it was renamed New York after King Charles II of England granted the lands to his brother, the Duke of York. The city has been continuously named New York since November 1674.',\n", - " 'source_documents': [Document(page_content='The city and its metropolitan area constitute the premier gateway for legal immigration to the United States. As many as 800 languages are spoken in New York, making it the most linguistically diverse city in the world. New York City is home to more than 3.2 million residents born outside the U.S., the largest foreign-born population of any city in the world as of 2016.New York City traces its origins to a trading post founded on the southern tip of Manhattan Island by Dutch colonists in approximately 1624. The settlement was named New Amsterdam (Dutch: Nieuw Amsterdam) in 1626 and was chartered as a city in 1653. The city came under British control in 1664 and was renamed New York after King Charles II of England granted the lands to his brother, the Duke of York. The city was regained by the Dutch in July 1673 and was renamed New Orange for one year and three months; the city has been continuously named New York since November 1674. New York City was the capital of the United States', metadata={'source': '../data/nyc_text.txt'}),\n", - " Document(page_content='New York City has been a metropolitan municipality with a Strong mayor–council form of government since its consolidation in 1898. In New York City, the city government is responsible for public education, correctional institutions, public safety, recreational facilities, sanitation, water supply, and welfare services.', metadata={'source': '../data/nyc_text.txt'}),\n", - " Document(page_content=\"Despite New York's heavy reliance on its vast public transit system, streets are a defining feature of the city. The Commissioners' Plan of 1811 greatly influenced the city's physical development. Several of the city's streets and avenues, including Broadway, Wall Street, Madison Avenue, and Seventh Avenue are also used as metonyms for national industries there: the theater, finance, advertising, and fashion organizations, respectively.\", metadata={'source': '../data/nyc_text.txt'}),\n", - " Document(page_content=\"In the pre-Columbian era, the area of present-day New York City was inhabited by Algonquian Native Americans, including the Lenape. Their homeland, known as Lenapehoking, included the present-day areas of Staten Island, Manhattan, the Bronx, the western portion of Long Island (including the areas that would later become the boroughs of Brooklyn and Queens), and the Lower Hudson Valley.The first documented visit into New York Harbor by a European was in 1524 by Italian Giovanni da Verrazzano, an explorer from Florence in the service of the French crown. He claimed the area for France and named it Nouvelle Angoulême (New Angoulême). A Spanish expedition, led by the Portuguese captain Estêvão Gomes sailing for Emperor Charles V, arrived in New York Harbor in January 1525 and charted the mouth of the Hudson River, which he named Río de San Antonio ('Saint Anthony's River'). The Padrón Real of 1527, the first scientific map to show the East Coast of North America continuously, was informed\", metadata={'source': '../data/nyc_text.txt'})]}" + " 'source_documents': [Document(id='a2202752-cd29-460a-a000-b8b873cc69cd', metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/nyc_text.txt'}, page_content='The city and its metropolitan area constitute the premier gateway for legal immigration to the United States. As many as 800 languages are spoken in New York, making it the most linguistically diverse city in the world. New York City is home to more than 3.2 million residents born outside the U.S., the largest foreign-born population of any city in the world as of 2016.New York City traces its origins to a trading post founded on the southern tip of Manhattan Island by Dutch colonists in approximately 1624. The settlement was named New Amsterdam (Dutch: Nieuw Amsterdam) in 1626 and was chartered as a city in 1653. The city came under British control in 1664 and was renamed New York after King Charles II of England granted the lands to his brother, the Duke of York. The city was regained by the Dutch in July 1673 and was renamed New Orange for one year and three months; the city has been continuously named New York since November 1674. New York City was the capital of the United States'),\n", + " Document(id='6467b9fc-11f8-4ecc-8fe1-59e35fdc0c6c', metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/nyc_text.txt'}, page_content='New York City has been a metropolitan municipality with a Strong mayor–council form of government since its consolidation in 1898. In New York City, the city government is responsible for public education, correctional institutions, public safety, recreational facilities, sanitation, water supply, and welfare services.'),\n", + " Document(id='324c12e1-48c1-454f-8f17-39c3ae6f6b26', metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/nyc_text.txt'}, page_content=\"Despite New York's heavy reliance on its vast public transit system, streets are a defining feature of the city. The Commissioners' Plan of 1811 greatly influenced the city's physical development. Several of the city's streets and avenues, including Broadway, Wall Street, Madison Avenue, and Seventh Avenue are also used as metonyms for national industries there: the theater, finance, advertising, and fashion organizations, respectively.\"),\n", + " Document(id='ef5b645b-9ce3-47f6-9f7f-54f2f1fa8eae', metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/nyc_text.txt'}, page_content=\"In the pre-Columbian era, the area of present-day New York City was inhabited by Algonquian Native Americans, including the Lenape. Their homeland, known as Lenapehoking, included the present-day areas of Staten Island, Manhattan, the Bronx, the western portion of Long Island (including the areas that would later become the boroughs of Brooklyn and Queens), and the Lower Hudson Valley.The first documented visit into New York Harbor by a European was in 1524 by Italian Giovanni da Verrazzano, an explorer from Florence in the service of the French crown. He claimed the area for France and named it Nouvelle Angoulême (New Angoulême). A Spanish expedition, led by the Portuguese captain Estêvão Gomes sailing for Emperor Charles V, arrived in New York Harbor in January 1525 and charted the mouth of the Hudson River, which he named Río de San Antonio ('Saint Anthony's River'). The Padrón Real of 1527, the first scientific map to show the East Coast of North America continuously, was informed\")]}" ] }, - "execution_count": 28, + "execution_count": 41, "metadata": {}, "output_type": "execute_result" } @@ -1115,7 +1196,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 42, "id": "a2e2cade-0005-41c1-b775-c6a7175bcf3b", "metadata": { "tags": [] @@ -1146,7 +1227,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 43, "id": "aac9358e-f8bc-4992-aea3-c83160ff0ab0", "metadata": { "tags": [] @@ -1167,7 +1248,7 @@ " 'ground_truths': ['The Statue of Liberty in New York City holds great significance as a symbol of the United States and its ideals of liberty and peace. It greeted millions of immigrants who arrived in the U.S. by ship in the late 19th and early 20th centuries, representing hope and freedom for those seeking a better life. It has since become an iconic landmark and a global symbol of cultural diversity and freedom.']}]" ] }, - "execution_count": 33, + "execution_count": 43, "metadata": {}, "output_type": "execute_result" } @@ -1202,7 +1283,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 44, "id": "022c8aae-fe5f-4274-b638-9209151b9491", "metadata": { "tags": [] @@ -1214,7 +1295,7 @@ "'Manhattan (New York County) has the highest population density of any borough in New York City.'" ] }, - "execution_count": 37, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } @@ -1226,7 +1307,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 45, "id": "eae31c80-42c9-4b1f-95b3-c05ceadb103f", "metadata": { "tags": [] @@ -1247,7 +1328,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 46, "id": "ecd4dd9f-16d7-43d4-ac8e-6c5aa5e3f7b0", "metadata": { "tags": [] @@ -1258,13 +1339,13 @@ "text/plain": [ "{'question': 'Which borough of New York City has the highest population?',\n", " 'answer': 'Manhattan (New York County) has the highest population density of any borough in New York City.',\n", - " 'contexts': [Document(page_content=\"New York City is the most populous city in the United States, with 8,804,190 residents incorporating more immigration into the city than outmigration since the 2010 United States census. More than twice as many people live in New York City as compared to Los Angeles, the second-most populous U.S. city; and New York has more than three times the population of Chicago, the third-most populous U.S. city. New York City gained more residents between 2010 and 2020 (629,000) than any other U.S. city, and a greater amount than the total sum of the gains over the same decade of the next four largest U.S. cities, Los Angeles, Chicago, Houston, and Phoenix, Arizona combined. New York City's population is about 44% of New York State's population, and about 39% of the population of the New York metropolitan area. The majority of New York City residents in 2020 (5,141,538, or 58.4%) were living on Long Island, in Brooklyn, or in Queens. The New York City metropolitan statistical area, has the\", metadata={'source': '../data/nyc_text.txt'}),\n", - " Document(page_content=\"New York, often called New York City or NYC, is the most populous city in the United States. With a 2020 population of 8,804,190 distributed over 300.46 square miles (778.2 km2), New York City is the most densely populated major city in the United States and more than twice as populous as Los Angeles, the nation's second-largest city. New York City is located at the southern tip of New York State. It constitutes the geographical and demographic center of both the Northeast megalopolis and the New York metropolitan area, the largest metropolitan area in the U.S. by both population and urban area. With over 20.1 million people in its metropolitan statistical area and 23.5 million in its combined statistical area as of 2020, New York is one of the world's most populous megacities, and over 58 million people live within 250 mi (400 km) of the city. New York City is a global cultural, financial, entertainment, and media center with a significant influence on commerce, health care and life\", metadata={'source': '../data/nyc_text.txt'}),\n", - " Document(page_content=\"Manhattan (New York County) is the geographically smallest and most densely populated borough. It is home to Central Park and most of the city's skyscrapers, and is sometimes locally known as The City. Manhattan's population density of 72,033 people per square mile (27,812/km2) in 2015 makes it the highest of any county in the United States and higher than the density of any individual American city.Manhattan is the cultural, administrative, and financial center of New York City and contains the headquarters of many major multinational corporations, the United Nations headquarters, Wall Street, and a number of important universities. The borough of Manhattan is often described as the financial and cultural center of the world.Most of the borough is situated on Manhattan Island, at the mouth of the Hudson River and the East River, and its southern tip, at the confluence of the two rivers, represents the birthplace of New York City itself. Several small islands also compose part of the\", metadata={'source': '../data/nyc_text.txt'}),\n", - " Document(page_content=\"=== Population density ===\\n\\nIn 2020, the city had an estimated population density of 29,302.37 inhabitants per square mile (11,313.71/km2), rendering it the nation's most densely populated of all larger municipalities (those with more than 100,000 residents), with several small cities (of fewer than 100,000) in adjacent Hudson County, New Jersey having greater density, as per the 2010 census. Geographically co-extensive with New York County, the borough of Manhattan's 2017 population density of 72,918 inhabitants per square mile (28,154/km2) makes it the highest of any county in the United States and higher than the density of any individual American city. The next three densest counties in the United States, placing second through fourth, are also New York boroughs: Brooklyn, the Bronx, and Queens respectively.\\n\\n\\n=== Race and ethnicity ===\", metadata={'source': '../data/nyc_text.txt'})]}" + " 'contexts': [Document(id='c8713903-e5b5-4683-8b44-bc7bc89b0c5d', metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/nyc_text.txt'}, page_content=\"New York City is the most populous city in the United States, with 8,804,190 residents incorporating more immigration into the city than outmigration since the 2010 United States census. More than twice as many people live in New York City as compared to Los Angeles, the second-most populous U.S. city; and New York has more than three times the population of Chicago, the third-most populous U.S. city. New York City gained more residents between 2010 and 2020 (629,000) than any other U.S. city, and a greater amount than the total sum of the gains over the same decade of the next four largest U.S. cities, Los Angeles, Chicago, Houston, and Phoenix, Arizona combined. New York City's population is about 44% of New York State's population, and about 39% of the population of the New York metropolitan area. The majority of New York City residents in 2020 (5,141,538, or 58.4%) were living on Long Island, in Brooklyn, or in Queens. The New York City metropolitan statistical area, has the\"),\n", + " Document(id='dd487d93-0eeb-4ecd-b496-b4f998cfdc72', metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/nyc_text.txt'}, page_content=\"New York, often called New York City or NYC, is the most populous city in the United States. With a 2020 population of 8,804,190 distributed over 300.46 square miles (778.2 km2), New York City is the most densely populated major city in the United States and more than twice as populous as Los Angeles, the nation's second-largest city. New York City is located at the southern tip of New York State. It constitutes the geographical and demographic center of both the Northeast megalopolis and the New York metropolitan area, the largest metropolitan area in the U.S. by both population and urban area. With over 20.1 million people in its metropolitan statistical area and 23.5 million in its combined statistical area as of 2020, New York is one of the world's most populous megacities, and over 58 million people live within 250 mi (400 km) of the city. New York City is a global cultural, financial, entertainment, and media center with a significant influence on commerce, health care and life\"),\n", + " Document(id='7f8d6b48-ee72-41dd-b781-6c9f10873d0d', metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/nyc_text.txt'}, page_content=\"Manhattan (New York County) is the geographically smallest and most densely populated borough. It is home to Central Park and most of the city's skyscrapers, and is sometimes locally known as The City. Manhattan's population density of 72,033 people per square mile (27,812/km2) in 2015 makes it the highest of any county in the United States and higher than the density of any individual American city.Manhattan is the cultural, administrative, and financial center of New York City and contains the headquarters of many major multinational corporations, the United Nations headquarters, Wall Street, and a number of important universities. The borough of Manhattan is often described as the financial and cultural center of the world.Most of the borough is situated on Manhattan Island, at the mouth of the Hudson River and the East River, and its southern tip, at the confluence of the two rivers, represents the birthplace of New York City itself. Several small islands also compose part of the\"),\n", + " Document(id='febd15fa-d871-4ebf-b676-fced2aa2b47b', metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/nyc_text.txt'}, page_content=\"=== Population density ===\\n\\nIn 2020, the city had an estimated population density of 29,302.37 inhabitants per square mile (11,313.71/km2), rendering it the nation's most densely populated of all larger municipalities (those with more than 100,000 residents), with several small cities (of fewer than 100,000) in adjacent Hudson County, New Jersey having greater density, as per the 2010 census. Geographically co-extensive with New York County, the borough of Manhattan's 2017 population density of 72,918 inhabitants per square mile (28,154/km2) makes it the highest of any county in the United States and higher than the density of any individual American city. The next three densest counties in the United States, placing second through fourth, are also New York boroughs: Brooklyn, the Bronx, and Queens respectively.\\n\\n\\n=== Race and ethnicity ===\")]}" ] }, - "execution_count": 53, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } @@ -1275,36 +1356,179 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 47, "id": "00d5cc30-d25e-41bd-8695-9246b73938bc", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Defaulting to user installation because normal site-packages is not writeable\n", + "Collecting recordclass\n", + " Downloading recordclass-0.22.1.tar.gz (1.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25h Installing build dependencies ... \u001b[?25ldone\n", + "\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n", + "\u001b[?25h Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n", + "\u001b[?25hBuilding wheels for collected packages: recordclass\n", + " Building wheel for recordclass (pyproject.toml) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for recordclass: filename=recordclass-0.22.1-cp312-cp312-macosx_11_0_arm64.whl size=126134 sha256=9b8ade7ba632b73101ac92b2ed3ef152ec4869667622fa8b8d3eb31319f10a15\n", + " Stored in directory: /private/var/folders/qp/3cxfqxvd0dq7rn_xc3tvtcz00000gp/T/pip-ephem-wheel-cache-rdjoudzf/wheels/51/f9/65/9666a410b46b914e738a0ebe2a81fc076c778fdb6a3359e539\n", + "Successfully built recordclass\n", + "Installing collected packages: recordclass\n", + "Successfully installed recordclass-0.22.1\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" + ] + } + ], "source": [ - "# !pip install --no-cache-dir recordclass" + "#!pip install --no-cache-dir recordclass" ] }, { "cell_type": "code", - "execution_count": 158, + "execution_count": 48, "id": "775d55d9-437d-40c4-bb5f-87c7b65fa567", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Defaulting to user installation because normal site-packages is not writeable\n", + "Collecting ragas==0.1.9\n", + " Downloading ragas-0.1.9-py3-none-any.whl.metadata (5.2 kB)\n", + "Requirement already satisfied: numpy in /Users/zone/.local/lib/python3.12/site-packages (from ragas==0.1.9) (1.26.4)\n", + "Requirement already satisfied: datasets in /Users/zone/.local/lib/python3.12/site-packages (from ragas==0.1.9) (3.0.2)\n", + "Requirement already satisfied: tiktoken in /Users/zone/.local/lib/python3.12/site-packages (from ragas==0.1.9) (0.8.0)\n", + "Requirement already satisfied: langchain in /Users/zone/.local/lib/python3.12/site-packages (from ragas==0.1.9) (0.3.4)\n", + "Requirement already satisfied: langchain-core in /Users/zone/.local/lib/python3.12/site-packages (from ragas==0.1.9) (0.3.13)\n", + "Requirement already satisfied: langchain-community in /Users/zone/.local/lib/python3.12/site-packages (from ragas==0.1.9) (0.3.3)\n", + "Requirement already satisfied: langchain-openai in /Users/zone/.local/lib/python3.12/site-packages (from ragas==0.1.9) (0.2.3)\n", + "Requirement already satisfied: openai>1 in /Users/zone/.local/lib/python3.12/site-packages (from ragas==0.1.9) (1.52.2)\n", + "Collecting pysbd>=0.3.4 (from ragas==0.1.9)\n", + " Downloading pysbd-0.3.4-py3-none-any.whl.metadata (6.1 kB)\n", + "Requirement already satisfied: nest-asyncio in /opt/anaconda3/lib/python3.12/site-packages (from ragas==0.1.9) (1.6.0)\n", + "Requirement already satisfied: appdirs in /opt/anaconda3/lib/python3.12/site-packages (from ragas==0.1.9) (1.4.4)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /opt/anaconda3/lib/python3.12/site-packages (from openai>1->ragas==0.1.9) (4.2.0)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /opt/anaconda3/lib/python3.12/site-packages (from openai>1->ragas==0.1.9) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /Users/zone/.local/lib/python3.12/site-packages (from openai>1->ragas==0.1.9) (0.27.2)\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/zone/.local/lib/python3.12/site-packages (from openai>1->ragas==0.1.9) (0.5.0)\n", + "Requirement already satisfied: pydantic<3,>=1.9.0 in /Users/zone/.local/lib/python3.12/site-packages (from openai>1->ragas==0.1.9) (2.9.2)\n", + "Requirement already satisfied: sniffio in /opt/anaconda3/lib/python3.12/site-packages (from openai>1->ragas==0.1.9) (1.3.0)\n", + "Requirement already satisfied: tqdm>4 in /opt/anaconda3/lib/python3.12/site-packages (from openai>1->ragas==0.1.9) (4.66.4)\n", + "Requirement already satisfied: typing-extensions<5,>=4.11 in /opt/anaconda3/lib/python3.12/site-packages (from openai>1->ragas==0.1.9) (4.11.0)\n", + "Requirement already satisfied: filelock in /opt/anaconda3/lib/python3.12/site-packages (from datasets->ragas==0.1.9) (3.13.1)\n", + "Requirement already satisfied: pyarrow>=15.0.0 in /Users/zone/.local/lib/python3.12/site-packages (from datasets->ragas==0.1.9) (17.0.0)\n", + "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /Users/zone/.local/lib/python3.12/site-packages (from datasets->ragas==0.1.9) (0.3.6)\n", + "Requirement already satisfied: pandas in /opt/anaconda3/lib/python3.12/site-packages (from datasets->ragas==0.1.9) (2.2.2)\n", + "Requirement already satisfied: requests>=2.32.2 in /opt/anaconda3/lib/python3.12/site-packages (from datasets->ragas==0.1.9) (2.32.2)\n", + "Requirement already satisfied: xxhash in /Users/zone/.local/lib/python3.12/site-packages (from datasets->ragas==0.1.9) (3.5.0)\n", + "Requirement already satisfied: multiprocess<0.70.17 in /Users/zone/.local/lib/python3.12/site-packages (from datasets->ragas==0.1.9) (0.70.14)\n", + "Requirement already satisfied: fsspec<=2024.9.0,>=2023.1.0 in /Users/zone/.local/lib/python3.12/site-packages (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets->ragas==0.1.9) (2024.2.0)\n", + "Requirement already satisfied: aiohttp in /opt/anaconda3/lib/python3.12/site-packages (from datasets->ragas==0.1.9) (3.9.5)\n", + "Requirement already satisfied: huggingface-hub>=0.23.0 in /Users/zone/.local/lib/python3.12/site-packages (from datasets->ragas==0.1.9) (0.25.1)\n", + "Requirement already satisfied: packaging in /opt/anaconda3/lib/python3.12/site-packages (from datasets->ragas==0.1.9) (23.2)\n", + "Requirement already satisfied: pyyaml>=5.1 in /opt/anaconda3/lib/python3.12/site-packages (from datasets->ragas==0.1.9) (6.0.1)\n", + "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /opt/anaconda3/lib/python3.12/site-packages (from langchain->ragas==0.1.9) (2.0.30)\n", + "Requirement already satisfied: langchain-text-splitters<0.4.0,>=0.3.0 in /Users/zone/.local/lib/python3.12/site-packages (from langchain->ragas==0.1.9) (0.3.0)\n", + "Requirement already satisfied: langsmith<0.2.0,>=0.1.17 in /Users/zone/.local/lib/python3.12/site-packages (from langchain->ragas==0.1.9) (0.1.137)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10,>=8.1.0 in /Users/zone/.local/lib/python3.12/site-packages (from langchain->ragas==0.1.9) (8.5.0)\n", + "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /opt/anaconda3/lib/python3.12/site-packages (from langchain-core->ragas==0.1.9) (1.33)\n", + "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /Users/zone/.local/lib/python3.12/site-packages (from langchain-community->ragas==0.1.9) (0.6.7)\n", + "Requirement already satisfied: pydantic-settings<3.0.0,>=2.4.0 in /Users/zone/.local/lib/python3.12/site-packages (from langchain-community->ragas==0.1.9) (2.5.2)\n", + "Requirement already satisfied: regex>=2022.1.18 in /opt/anaconda3/lib/python3.12/site-packages (from tiktoken->ragas==0.1.9) (2023.10.3)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /opt/anaconda3/lib/python3.12/site-packages (from aiohttp->datasets->ragas==0.1.9) (1.2.0)\n", + "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda3/lib/python3.12/site-packages (from aiohttp->datasets->ragas==0.1.9) (23.1.0)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /opt/anaconda3/lib/python3.12/site-packages (from aiohttp->datasets->ragas==0.1.9) (1.4.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/anaconda3/lib/python3.12/site-packages (from aiohttp->datasets->ragas==0.1.9) (6.0.4)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/anaconda3/lib/python3.12/site-packages (from aiohttp->datasets->ragas==0.1.9) (1.9.3)\n", + "Requirement already satisfied: idna>=2.8 in /opt/anaconda3/lib/python3.12/site-packages (from anyio<5,>=3.5.0->openai>1->ragas==0.1.9) (3.7)\n", + "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /Users/zone/.local/lib/python3.12/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community->ragas==0.1.9) (3.22.0)\n", + "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /Users/zone/.local/lib/python3.12/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community->ragas==0.1.9) (0.9.0)\n", + "Requirement already satisfied: certifi in /opt/anaconda3/lib/python3.12/site-packages (from httpx<1,>=0.23.0->openai>1->ragas==0.1.9) (2024.7.4)\n", + "Requirement already satisfied: httpcore==1.* in /opt/anaconda3/lib/python3.12/site-packages (from httpx<1,>=0.23.0->openai>1->ragas==0.1.9) (1.0.2)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /opt/anaconda3/lib/python3.12/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai>1->ragas==0.1.9) (0.14.0)\n", + "Requirement already satisfied: jsonpointer>=1.9 in /opt/anaconda3/lib/python3.12/site-packages (from jsonpatch<2.0,>=1.33->langchain-core->ragas==0.1.9) (2.1)\n", + "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /Users/zone/.local/lib/python3.12/site-packages (from langsmith<0.2.0,>=0.1.17->langchain->ragas==0.1.9) (3.10.7)\n", + "Requirement already satisfied: requests-toolbelt<2.0.0,>=1.0.0 in /opt/anaconda3/lib/python3.12/site-packages (from langsmith<0.2.0,>=0.1.17->langchain->ragas==0.1.9) (1.0.0)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/lib/python3.12/site-packages (from pydantic<3,>=1.9.0->openai>1->ragas==0.1.9) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.23.4 in /Users/zone/.local/lib/python3.12/site-packages (from pydantic<3,>=1.9.0->openai>1->ragas==0.1.9) (2.23.4)\n", + "Requirement already satisfied: python-dotenv>=0.21.0 in /Users/zone/.local/lib/python3.12/site-packages (from pydantic-settings<3.0.0,>=2.4.0->langchain-community->ragas==0.1.9) (1.0.1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/lib/python3.12/site-packages (from requests>=2.32.2->datasets->ragas==0.1.9) (2.0.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/zone/.local/lib/python3.12/site-packages (from requests>=2.32.2->datasets->ragas==0.1.9) (2.2.3)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/zone/.local/lib/python3.12/site-packages (from pandas->datasets->ragas==0.1.9) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/lib/python3.12/site-packages (from pandas->datasets->ragas==0.1.9) (2024.1)\n", + "Requirement already satisfied: tzdata>=2022.7 in /opt/anaconda3/lib/python3.12/site-packages (from pandas->datasets->ragas==0.1.9) (2023.3)\n", + "Requirement already satisfied: six>=1.5 in /opt/anaconda3/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas->datasets->ragas==0.1.9) (1.16.0)\n", + "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/anaconda3/lib/python3.12/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain-community->ragas==0.1.9) (1.0.0)\n", + "Downloading ragas-0.1.9-py3-none-any.whl (86 kB)\n", + "Downloading pysbd-0.3.4-py3-none-any.whl (71 kB)\n", + "Installing collected packages: pysbd, ragas\n", + "Successfully installed pysbd-0.3.4 ragas-0.1.9\n" + ] + } + ], "source": [ - "# !pip install ragas==0.1.9" + "#!pip install ragas==0.1.9" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 49, "id": "29b14c07-bf6c-4e86-ad1a-2a6c4d1a509d", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/zone/.local/lib/python3.12/site-packages/ragas/metrics/__init__.py:1: LangChainDeprecationWarning: As of langchain-core 0.3.0, LangChain uses pydantic v2 internally. The langchain_core.pydantic_v1 module was a compatibility shim for pydantic v1, and should no longer be used. Please update the code to import from Pydantic directly.\n", + "\n", + "For example, replace imports like: `from langchain_core.pydantic_v1 import BaseModel`\n", + "with: `from pydantic import BaseModel`\n", + "or the v1 compatibility namespace if you are working in a code base that has not been fully upgraded to pydantic 2 yet. \tfrom pydantic.v1 import BaseModel\n", + "\n", + " from ragas.metrics._answer_correctness import AnswerCorrectness, answer_correctness\n", + "/Users/zone/.local/lib/python3.12/site-packages/ragas/metrics/__init__.py:4: LangChainDeprecationWarning: As of langchain-core 0.3.0, LangChain uses pydantic v2 internally. The langchain.pydantic_v1 module was a compatibility shim for pydantic v1, and should no longer be used. Please update the code to import from Pydantic directly.\n", + "\n", + "For example, replace imports like: `from langchain.pydantic_v1 import BaseModel`\n", + "with: `from pydantic import BaseModel`\n", + "or the v1 compatibility namespace if you are working in a code base that has not been fully upgraded to pydantic 2 yet. \tfrom pydantic.v1 import BaseModel\n", + "\n", + " from ragas.metrics._context_entities_recall import (\n" + ] + } + ], "source": [ "from ragas.integrations.langchain import EvaluatorChain \n", "# from ragas import evaluate\n", @@ -1334,143 +1558,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 50, "id": "7815671c-1fc8-46ba-8356-4a0bd5558530", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'query': 'Which borough of New York City has the highest population?',\n", + " 'result': 'Manhattan (New York County) has the highest population density of any borough in New York City.',\n", + " 'source_documents': [Document(id='c8713903-e5b5-4683-8b44-bc7bc89b0c5d', metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/nyc_text.txt'}, page_content=\"New York City is the most populous city in the United States, with 8,804,190 residents incorporating more immigration into the city than outmigration since the 2010 United States census. More than twice as many people live in New York City as compared to Los Angeles, the second-most populous U.S. city; and New York has more than three times the population of Chicago, the third-most populous U.S. city. New York City gained more residents between 2010 and 2020 (629,000) than any other U.S. city, and a greater amount than the total sum of the gains over the same decade of the next four largest U.S. cities, Los Angeles, Chicago, Houston, and Phoenix, Arizona combined. New York City's population is about 44% of New York State's population, and about 39% of the population of the New York metropolitan area. The majority of New York City residents in 2020 (5,141,538, or 58.4%) were living on Long Island, in Brooklyn, or in Queens. The New York City metropolitan statistical area, has the\"),\n", + " Document(id='dd487d93-0eeb-4ecd-b496-b4f998cfdc72', metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/nyc_text.txt'}, page_content=\"New York, often called New York City or NYC, is the most populous city in the United States. With a 2020 population of 8,804,190 distributed over 300.46 square miles (778.2 km2), New York City is the most densely populated major city in the United States and more than twice as populous as Los Angeles, the nation's second-largest city. New York City is located at the southern tip of New York State. It constitutes the geographical and demographic center of both the Northeast megalopolis and the New York metropolitan area, the largest metropolitan area in the U.S. by both population and urban area. With over 20.1 million people in its metropolitan statistical area and 23.5 million in its combined statistical area as of 2020, New York is one of the world's most populous megacities, and over 58 million people live within 250 mi (400 km) of the city. New York City is a global cultural, financial, entertainment, and media center with a significant influence on commerce, health care and life\"),\n", + " Document(id='7f8d6b48-ee72-41dd-b781-6c9f10873d0d', metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/nyc_text.txt'}, page_content=\"Manhattan (New York County) is the geographically smallest and most densely populated borough. It is home to Central Park and most of the city's skyscrapers, and is sometimes locally known as The City. Manhattan's population density of 72,033 people per square mile (27,812/km2) in 2015 makes it the highest of any county in the United States and higher than the density of any individual American city.Manhattan is the cultural, administrative, and financial center of New York City and contains the headquarters of many major multinational corporations, the United Nations headquarters, Wall Street, and a number of important universities. The borough of Manhattan is often described as the financial and cultural center of the world.Most of the borough is situated on Manhattan Island, at the mouth of the Hudson River and the East River, and its southern tip, at the confluence of the two rivers, represents the birthplace of New York City itself. Several small islands also compose part of the\"),\n", + " Document(id='febd15fa-d871-4ebf-b676-fced2aa2b47b', metadata={'source': '/Users/zone/Desktop/Week 8/D2/lab-langchain-evaluation/data/nyc_text.txt'}, page_content=\"=== Population density ===\\n\\nIn 2020, the city had an estimated population density of 29,302.37 inhabitants per square mile (11,313.71/km2), rendering it the nation's most densely populated of all larger municipalities (those with more than 100,000 residents), with several small cities (of fewer than 100,000) in adjacent Hudson County, New Jersey having greater density, as per the 2010 census. Geographically co-extensive with New York County, the borough of Manhattan's 2017 population density of 72,918 inhabitants per square mile (28,154/km2) makes it the highest of any county in the United States and higher than the density of any individual American city. The next three densest counties in the United States, placing second through fourth, are also New York boroughs: Brooklyn, the Bronx, and Queens respectively.\\n\\n\\n=== Race and ethnicity ===\")]}" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Recheck the result that we are going to validate.\n", "result" ] - }, - { - "cell_type": "markdown", - "id": "8cfdd07c-956c-458f-8844-e056f29e3c83", - "metadata": {}, - "source": [ - "**Faithfulness**" - ] - }, - { - "cell_type": "code", - "execution_count": 87, - "id": "304f5f0f-a237-4584-becb-a35607caf26b", - "metadata": {}, - "outputs": [], - "source": [ - "eval_result = faithfulness_chain(result_updated)\n", - "eval_result[\"faithfulness_score\"]" - ] - }, - { - "cell_type": "markdown", - "id": "8fbd78a9-d7c5-42a0-8705-4c544ec6408e", - "metadata": {}, - "source": [ - "High faithfulness_score means that there are exact consistency between the source documents and the answer.\n", - "\n", - "You can check lower faithfulness scores by changing the result (answer from LLM) or source_documents to something else." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15f42b97-a84f-4015-9da5-fa3b0b703c24", - "metadata": {}, - "outputs": [], - "source": [ - "fake_result = result.copy()\n", - "fake_result[\"result\"] = \"we are the champions\"\n", - "eval_result = faithfulness_chain(fake_result)\n", - "eval_result[\"faithfulness_score\"]" - ] - }, - { - "cell_type": "markdown", - "id": "02348380-159a-4578-a95e-493cdcfcebe7", - "metadata": {}, - "source": [ - "**Context Relevancy**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0fd2fb9e-19d1-4cf6-9773-897546c2d6bb", - "metadata": {}, - "outputs": [], - "source": [ - "eval_result = context_recall_chain(result)\n", - "eval_result[\"context_recall_score\"]" - ] - }, - { - "cell_type": "markdown", - "id": "118cba80-c2a3-408e-9d8d-857521cbe723", - "metadata": {}, - "source": [ - "High context_recall_score means that the ground truth is present in the source documents.\n", - "\n", - "You can check lower context recall scores by changing the source_documents to something else." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f6610c5f-5f8e-406c-885c-093012a5dc44", - "metadata": {}, - "outputs": [], - "source": [ - "from langchain.schema import Document\n", - "fake_result = result.copy()\n", - "fake_result[\"source_documents\"] = [Document(page_content=\"I love christmas\")]\n", - "eval_result = context_recall_chain(fake_result)\n", - "eval_result[\"context_recall_score\"]" - ] - }, - { - "cell_type": "markdown", - "id": "3dc06403-c514-47fd-ac82-d95ece8d2f06", - "metadata": {}, - "source": [ - "2. `evaluate()`\n", - "\n", - "Evaluate a list of inputs/queries and the outputs/predictions from the QA chain." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f6b23708-b94d-4649-acd9-0e268f72f94a", - "metadata": {}, - "outputs": [], - "source": [ - "# run the queries as a batch for efficiency\n", - "predictions = qa_chain.batch(examples)\n", - "\n", - "# evaluate\n", - "print(\"evaluating...\")\n", - "r = faithfulness_chain.evaluate(examples, predictions)\n", - "r" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d661f6b9-04c7-40b7-874b-25f53cfab9d9", - "metadata": {}, - "outputs": [], - "source": [ - "# evaluate context recall\n", - "print(\"evaluating...\")\n", - "r = context_recall_chain.evaluate(examples, predictions)\n", - "r" - ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "base", "language": "python", "name": "python3" }, @@ -1484,7 +1600,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.8" + "version": "3.12.4" } }, "nbformat": 4,