Skip to content

Commit

Permalink
tutorial46
Browse files Browse the repository at this point in the history
  • Loading branch information
@ronidas39
ronidas39 committed Mar 15, 2024
1 parent fb3664a commit d019120
Show file tree
Hide file tree
Showing 8 changed files with 384 additions and 2 deletions.
Binary file modified .DS_Store
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion tutorial2/single_url.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from langchain.document_loaders import youtube
import io

loader=youtube.YoutubeLoader.from_youtube_url("https://youtu.be/QcWtGoDLEz4")
loader=youtube.YoutubeLoader.from_youtube_url("https://youtu.be/YNb-aF733ew")
docs=loader.load()
print(docs)
with io.open("transcript.txt","w",encoding="utf-8")as f1:
Expand Down
2 changes: 1 addition & 1 deletion tutorial2/transcript.txt
View file

Large diffs are not rendered by default.

Binary file modified tutorial45/.DS_Store
View file
Binary file not shown.
Binary file added tutorial46/.DS_Store
View file
Binary file not shown.
382 changes: 382 additions & 0 deletions tutorial46/tutorial46.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,382 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "c341c81e-5a1a-49b5-a574-7053c927f347",
"showTitle": false,
"title": ""
}
},
"outputs": [],
"source": [
"from langchain_experimental.agents.agent_toolkits import create_spark_dataframe_agent\n",
"from langchain_openai import ChatOpenAI\n",
"from pyspark.sql import SparkSession"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "8264862b-7516-461c-8fe6-b2a9adcff49f",
"showTitle": false,
"title": ""
}
},
"outputs": [],
"source": [
"spark=SparkSession.builder.getOrCreate()"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "747414d3-f6ee-4a8b-845c-9939c0994d87",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"output_type": "stream",
"text": [
"+--------------------+---------------------+-------------+-----------+----------+-----------+\n|tpep_pickup_datetime|tpep_dropoff_datetime|trip_distance|fare_amount|pickup_zip|dropoff_zip|\n+--------------------+---------------------+-------------+-----------+----------+-----------+\n| 2016-02-14 16:52:13| 2016-02-14 17:16:04| 4.94| 19.0| 10282| 10171|\n| 2016-02-04 18:44:19| 2016-02-04 18:46:00| 0.28| 3.5| 10110| 10110|\n| 2016-02-17 17:13:57| 2016-02-17 17:17:55| 0.7| 5.0| 10103| 10023|\n| 2016-02-18 10:36:07| 2016-02-18 10:41:45| 0.8| 6.0| 10022| 10017|\n| 2016-02-22 14:14:41| 2016-02-22 14:31:52| 4.51| 17.0| 10110| 10282|\n| 2016-02-05 06:45:02| 2016-02-05 06:50:26| 1.8| 7.0| 10009| 10065|\n| 2016-02-15 15:03:28| 2016-02-15 15:18:45| 2.58| 12.0| 10153| 10199|\n| 2016-02-25 19:09:26| 2016-02-25 19:24:50| 1.4| 11.0| 10112| 10069|\n| 2016-02-13 16:28:18| 2016-02-13 16:36:36| 1.21| 7.5| 10023| 10153|\n| 2016-02-14 00:03:48| 2016-02-14 00:10:24| 0.6| 6.0| 10012| 10003|\n| 2016-02-27 15:02:58| 2016-02-27 15:08:31| 2.02| 8.0| 10002| 11211|\n| 2016-02-17 07:52:40| 2016-02-17 08:01:21| 1.5| 8.0| 10019| 10199|\n| 2016-02-14 21:55:55| 2016-02-14 22:01:31| 0.93| 6.0| 10019| 10018|\n| 2016-02-05 22:27:07| 2016-02-05 22:39:44| 2.34| 10.5| 10110| 10014|\n| 2016-02-05 09:51:47| 2016-02-05 09:57:27| 0.91| 5.5| 10119| 10199|\n| 2016-02-21 11:15:39| 2016-02-21 11:40:24| 11.6| 33.5| 10019| 11371|\n| 2016-02-23 13:20:29| 2016-02-23 13:36:25| 1.4| 11.0| 10018| 10022|\n| 2016-02-24 13:07:46| 2016-02-24 13:26:13| 2.43| 13.0| 10065| 10119|\n| 2016-02-22 15:12:53| 2016-02-22 15:22:59| 1.2| 8.0| 10001| 10009|\n| 2016-02-05 09:40:07| 2016-02-05 09:49:09| 1.3| 7.5| 10003| 10013|\n+--------------------+---------------------+-------------+-----------+----------+-----------+\nonly showing top 20 rows\n\n"
]
}
],
"source": [
"df=spark.sql(\"select * from samples.nyctaxi.trips\")\n",
"df.show()"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "e70904f2-01d9-43ba-b710-ffa5ce1e78a9",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"21932"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.count()"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "0d4f7e7e-1e50-4e9c-891d-5c92afe92db1",
"showTitle": false,
"title": ""
}
},
"outputs": [],
"source": [
"llm=ChatOpenAI(model=\"gpt-4\",temperature=0)\n",
"agent=create_spark_dataframe_agent(llm=llm,df=df,verbose=True)"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "392c5748-1265-437a-b603-f67f37c92bdc",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"output_type": "stream",
"text": [
"\n\n\u001B[1m> Entering new AgentExecutor chain...\u001B[0m\n\u001B[32;1m\u001B[1;3mThought: To get the columns of a dataframe in PySpark, we can use the `columns` property of the dataframe.\nAction: python_repl_ast\nAction Input: df.columns\u001B[0m\nObservation: \u001B[36;1m\u001B[1;3m['tpep_pickup_datetime', 'tpep_dropoff_datetime', 'trip_distance', 'fare_amount', 'pickup_zip', 'dropoff_zip']\u001B[0m\nThought:\u001B[32;1m\u001B[1;3mI now know the columns available in the dataframe.\nFinal Answer: The columns available are 'tpep_pickup_datetime', 'tpep_dropoff_datetime', 'trip_distance', 'fare_amount', 'pickup_zip', 'dropoff_zip'.\u001B[0m\n\n\u001B[1m> Finished chain.\u001B[0m\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"\"The columns available are 'tpep_pickup_datetime', 'tpep_dropoff_datetime', 'trip_distance', 'fare_amount', 'pickup_zip', 'dropoff_zip'.\""
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"agent.run(\"what are the columns available?\")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "e05a787c-00d7-48a9-a542-a3de75a7b475",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"output_type": "stream",
"text": [
"\n\n\u001B[1m> Entering new AgentExecutor chain...\u001B[0m\n\u001B[32;1m\u001B[1;3mThought: To find the maximum trip distance, I need to use the `max()` function on the `trip_distance` column of the dataframe.\nAction: python_repl_ast\nAction Input: df.agg({\"trip_distance\": \"max\"}).collect()[0][0]\u001B[0m\nObservation: \u001B[36;1m\u001B[1;3m30.6\u001B[0m\nThought:\u001B[32;1m\u001B[1;3mThe maximum trip distance in the dataframe is 30.6 miles.\nFinal Answer: 30.6\u001B[0m\n\n\u001B[1m> Finished chain.\u001B[0m\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"'30.6'"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"agent.run(\"what is maximum trip distance\")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "c29f122b-6a61-438a-b549-2e4eb5fb3031",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"output_type": "stream",
"text": [
"\n\n\u001B[1m> Entering new AgentExecutor chain...\u001B[0m\n\u001B[32;1m\u001B[1;3mThought: To find the zip code with the maximum number of pickups, we need to group the dataframe by the 'pickup_zip' column and count the number of rows in each group. Then, we need to order the groups by the count in descending order and select the first row.\nAction: python_repl_ast\nAction Input: df.groupBy('pickup_zip').count().orderBy('count', ascending=False).first()\u001B[0m\nObservation: \u001B[36;1m\u001B[1;3mRow(pickup_zip=10001, count=1227)\u001B[0m\nThought:\u001B[32;1m\u001B[1;3mThe zip code with the maximum number of pickups is 10001.\nFinal Answer: 10001\u001B[0m\n\n\u001B[1m> Finished chain.\u001B[0m\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"'10001'"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"agent.run(\"which zip code has maxiumum number of pick up\")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "9c18762b-8930-421e-8e96-0e19ee2e5d85",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"output_type": "stream",
"text": [
"\n\n\u001B[1m> Entering new AgentExecutor chain...\u001B[0m\n\u001B[32;1m\u001B[1;3mThought: To find the zip code with the maximum number of drops, we need to group the dataframe by the 'dropoff_zip' column and count the number of rows in each group. Then, we can sort the result in descending order and select the first row.\nAction: python_repl_ast\nAction Input: df.groupBy('dropoff_zip').count().orderBy('count', ascending=False).first()\u001B[0m\nObservation: \u001B[36;1m\u001B[1;3mRow(dropoff_zip=10001, count=1069)\u001B[0m\nThought:\u001B[32;1m\u001B[1;3mThe zip code with the maximum number of drops is 10001.\nFinal Answer: 10001\u001B[0m\n\n\u001B[1m> Finished chain.\u001B[0m\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"'10001'"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"agent.run(\"which zip code has maxiumum number of drop\")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "13af0679-f043-4743-a291-4fa672f55901",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"output_type": "stream",
"text": [
"\n\n\u001B[1m> Entering new AgentExecutor chain...\u001B[0m\n\u001B[32;1m\u001B[1;3mThought: To find the date with the maximum number of drops, we need to group the data by the dropoff date and then count the number of rows for each date. The date with the maximum count is the date with the maximum number of drops.\nAction: python_repl_ast\nAction Input: df.groupBy(df.tpep_dropoff_datetime).count().orderBy('count', ascending=False).first()\u001B[0m\nObservation: \u001B[36;1m\u001B[1;3mRow(tpep_dropoff_datetime=datetime.datetime(2016, 2, 14, 18, 25, 41), count=2)\u001B[0m\nThought:\u001B[32;1m\u001B[1;3mThe date with the maximum number of drops is 2016-02-14 18:25:41 with 2 drops.\nFinal Answer: The date with the maximum number of drops is 2016-02-14 18:25:41.\u001B[0m\n\n\u001B[1m> Finished chain.\u001B[0m\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"'The date with the maximum number of drops is 2016-02-14 18:25:41.'"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"agent.run(\"which date has maximum number of drop\")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "a39b43ca-226f-4c70-b8dd-a543ba4ae12e",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"output_type": "stream",
"text": [
"\n\n\u001B[1m> Entering new AgentExecutor chain...\u001B[0m\n\u001B[32;1m\u001B[1;3mThought: To find the maximum fare, I need to use the `max()` function on the `fare_amount` column of the dataframe.\nAction: python_repl_ast\nAction Input: df.agg({\"fare_amount\": \"max\"}).collect()\u001B[0m\nObservation: \u001B[36;1m\u001B[1;3m[Row(max(fare_amount)=275.0)]\u001B[0m\nThought:\u001B[32;1m\u001B[1;3mThe maximum fare amount for all trips is 275.0\nFinal Answer: 275.0\u001B[0m\n\n\u001B[1m> Finished chain.\u001B[0m\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"'275.0'"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"agent.run(\"what is the maximum fare considering all trips\")"
]
}
],
"metadata": {
"application/vnd.databricks.v1+notebook": {
"dashboards": [],
"language": "python",
"notebookMetadata": {
"pythonIndentUnit": 4
},
"notebookName": "Untitled Notebook 2024-03-15 23:21:17",
"widgets": {}
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Binary file added tutorial46/tutorial46.pptx
View file
Binary file not shown.
Binary file added tutorial46/~$tutorial46.pptx
View file
Binary file not shown.

0 comments on commit d019120

Please sign in to comment.