From 7ca7667409dc16e021a10826e3fd1bfb8995c2f5 Mon Sep 17 00:00:00 2001 From: Pathway-Dev <72749368+Pathway-Dev@users.noreply.github.com> Date: Tue, 14 Jan 2025 06:39:44 +0100 Subject: [PATCH] Daily Pathway examples refresh Co-authored-by: Manul from Pathway GitOrigin-RevId: a03734e74149ce9d0b6d11beba0313fd5bf030a0 --- .../live_vector_indexing_pipeline.ipynb | 4 +- .../tutorials/asynctransformer.ipynb | 110 +++++++++--------- examples/notebooks/tutorials/iterate.ipynb | 2 +- .../notebooks/tutorials/option-greeks.ipynb | 3 +- 4 files changed, 60 insertions(+), 59 deletions(-) diff --git a/examples/notebooks/showcases/live_vector_indexing_pipeline.ipynb b/examples/notebooks/showcases/live_vector_indexing_pipeline.ipynb index 02034f1a..8ba1b804 100644 --- a/examples/notebooks/showcases/live_vector_indexing_pipeline.ipynb +++ b/examples/notebooks/showcases/live_vector_indexing_pipeline.ipynb @@ -53,7 +53,7 @@ "source": [ "# Always Up-to-date Data Indexing pipeline\n", "\n", - "This showcase shows how to use Pathway to deploy a live data indexing pipeline, which can be queried like a typical vector store. However, under the hood, Pathway updates the index on each data change, always giving up-to-date answers.\n", + "This showcase demonstrates how to use Pathway to deploy a live data indexing pipeline that can be queried similarly to a typical vector store. Unlike traditional approaches, Pathway updates the index with every data change, ensuring consistently up-to-date answers.\n", "\n", "::article-img\n", "---\n", @@ -100,7 +100,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install pathway litellm\n", + "!pip install \"pathway[xpack-llm,xpack-llm-docs]\"\n", " !pip install unstructured[all-docs]\n", "!mkdir -p sample_documents\n", "![ -f sample_documents/repo_readme.md ] || wget 'https://gist.githubusercontent.com/janchorowski/dd22a293f3d99d1b726eedc7d46d2fc0/raw/pathway_readme.md' -O 'sample_documents/repo_readme.md'\n", diff --git a/examples/notebooks/tutorials/asynctransformer.ipynb b/examples/notebooks/tutorials/asynctransformer.ipynb index 9c28ac20..7d3cd363 100644 --- a/examples/notebooks/tutorials/asynctransformer.ipynb +++ b/examples/notebooks/tutorials/asynctransformer.ipynb @@ -162,11 +162,11 @@ "output_type": "stream", "text": [ " | value | ret | __time__ | __diff__\n", - "^Z3QWT29... | 2 | 3 | 1736746377326 | 1\n", - "^3CZ78B4... | 2 | 3 | 1736746377526 | 1\n", - "^YYY4HAB... | 6 | 7 | 1736746377526 | 1\n", - "^3HN31E1... | 6 | 7 | 1736746377928 | 1\n", - "^X1MXHYY... | 12 | 13 | 1736746377928 | 1\n" + "^Z3QWT29... | 2 | 3 | 1736832790750 | 1\n", + "^3CZ78B4... | 2 | 3 | 1736832790950 | 1\n", + "^YYY4HAB... | 6 | 7 | 1736832790950 | 1\n", + "^3HN31E1... | 6 | 7 | 1736832791352 | 1\n", + "^X1MXHYY... | 12 | 13 | 1736832791352 | 1\n" ] } ], @@ -410,12 +410,12 @@ "output_type": "stream", "text": [ " | group | value | ret | __time__ | __diff__\n", - "^Z3QWT29... | 2 | 1 | 2 | 1736746381330 | 1\n", - "^Z3QWT29... | 2 | 1 | 2 | 1736746381430 | -1\n", - "^Z3QWT29... | 2 | 3 | 4 | 1736746381430 | 1\n", - "^YYY4HAB... | 1 | 5 | 6 | 1736746381630 | 1\n", - "^YYY4HAB... | 1 | 5 | 6 | 1736746381832 | -1\n", - "^YYY4HAB... | 1 | 2 | 3 | 1736746381832 | 1\n" + "^Z3QWT29... | 2 | 1 | 2 | 1736832794996 | 1\n", + "^Z3QWT29... | 2 | 1 | 2 | 1736832795096 | -1\n", + "^Z3QWT29... | 2 | 3 | 4 | 1736832795096 | 1\n", + "^YYY4HAB... | 1 | 5 | 6 | 1736832795296 | 1\n", + "^YYY4HAB... | 1 | 5 | 6 | 1736832795498 | -1\n", + "^YYY4HAB... | 1 | 2 | 3 | 1736832795498 | 1\n" ] } ], @@ -537,18 +537,18 @@ "output_type": "stream", "text": [ " | group | value | ret | __time__ | __diff__\n", - "^Z3QWT29... | 2 | 1 | 2 | 1736746382070 | 1\n", - "^3HN31E1... | 4 | 3 | 4 | 1736746382070 | 1\n", - "^Z3QWT29... | 2 | 1 | 2 | 1736746382370 | -1\n", - "^3HN31E1... | 4 | 3 | 4 | 1736746382370 | -1\n", - "^Z3QWT29... | 2 | 4 | 5 | 1736746382370 | 1\n", - "^3HN31E1... | 4 | 2 | 3 | 1736746382370 | 1\n", - "^YYY4HAB... | 1 | 5 | 6 | 1736746382470 | 1\n", - "^3CZ78B4... | 3 | 1 | 2 | 1736746382470 | 1\n", - "^YYY4HAB... | 1 | 5 | 6 | 1736746382570 | -1\n", - "^3CZ78B4... | 3 | 1 | 2 | 1736746382570 | -1\n", - "^YYY4HAB... | 1 | 2 | 3 | 1736746382570 | 1\n", - "^3CZ78B4... | 3 | 2 | 3 | 1736746382570 | 1\n" + "^Z3QWT29... | 2 | 1 | 2 | 1736832795832 | 1\n", + "^3HN31E1... | 4 | 3 | 4 | 1736832795832 | 1\n", + "^Z3QWT29... | 2 | 1 | 2 | 1736832796134 | -1\n", + "^3HN31E1... | 4 | 3 | 4 | 1736832796134 | -1\n", + "^Z3QWT29... | 2 | 4 | 5 | 1736832796134 | 1\n", + "^3HN31E1... | 4 | 2 | 3 | 1736832796134 | 1\n", + "^YYY4HAB... | 1 | 5 | 6 | 1736832796234 | 1\n", + "^3CZ78B4... | 3 | 1 | 2 | 1736832796234 | 1\n", + "^YYY4HAB... | 1 | 5 | 6 | 1736832796332 | -1\n", + "^3CZ78B4... | 3 | 1 | 2 | 1736832796332 | -1\n", + "^YYY4HAB... | 1 | 2 | 3 | 1736832796332 | 1\n", + "^3CZ78B4... | 3 | 2 | 3 | 1736832796332 | 1\n" ] } ], @@ -581,16 +581,16 @@ "output_type": "stream", "text": [ " | group | value | ret | __time__ | __diff__\n", - "^Z3QWT29... | 2 | 1 | 2 | 1736746382706 | 1\n", - "^3CZ78B4... | 3 | 2 | 3 | 1736746382806 | 1\n", - "^3HN31E1... | 4 | 3 | 4 | 1736746382908 | 1\n", - "^Z3QWT29... | 2 | 1 | 2 | 1736746383008 | -1\n", - "^3HN31E1... | 4 | 3 | 4 | 1736746383008 | -1\n", - "^Z3QWT29... | 2 | 4 | 5 | 1736746383008 | 1\n", - "^3HN31E1... | 4 | 2 | 3 | 1736746383008 | 1\n", - "^YYY4HAB... | 1 | 5 | 6 | 1736746383108 | 1\n", - "^YYY4HAB... | 1 | 5 | 6 | 1736746383208 | -1\n", - "^YYY4HAB... | 1 | 2 | 3 | 1736746383208 | 1\n" + "^Z3QWT29... | 2 | 1 | 2 | 1736832796474 | 1\n", + "^3CZ78B4... | 3 | 2 | 3 | 1736832796574 | 1\n", + "^3HN31E1... | 4 | 3 | 4 | 1736832796674 | 1\n", + "^Z3QWT29... | 2 | 1 | 2 | 1736832796776 | -1\n", + "^3HN31E1... | 4 | 3 | 4 | 1736832796776 | -1\n", + "^Z3QWT29... | 2 | 4 | 5 | 1736832796776 | 1\n", + "^3HN31E1... | 4 | 2 | 3 | 1736832796776 | 1\n", + "^YYY4HAB... | 1 | 5 | 6 | 1736832796874 | 1\n", + "^YYY4HAB... | 1 | 5 | 6 | 1736832796974 | -1\n", + "^YYY4HAB... | 1 | 2 | 3 | 1736832796974 | 1\n" ] } ], @@ -633,18 +633,18 @@ "output_type": "stream", "text": [ " | group | value | ret | __time__ | __diff__\n", - "^YYY4HAB... | 1 | 5 | 6 | 1736746383344 | 1\n", - "^Z3QWT29... | 2 | 1 | 2 | 1736746383344 | 1\n", - "^3CZ78B4... | 3 | 1 | 2 | 1736746383344 | 1\n", - "^3HN31E1... | 4 | 3 | 4 | 1736746383344 | 1\n", - "^YYY4HAB... | 1 | 5 | 6 | 1736746383846 | -1\n", - "^Z3QWT29... | 2 | 1 | 2 | 1736746383846 | -1\n", - "^3CZ78B4... | 3 | 1 | 2 | 1736746383846 | -1\n", - "^3HN31E1... | 4 | 3 | 4 | 1736746383846 | -1\n", - "^YYY4HAB... | 1 | 2 | 3 | 1736746383846 | 1\n", - "^Z3QWT29... | 2 | 4 | 5 | 1736746383846 | 1\n", - "^3CZ78B4... | 3 | 2 | 3 | 1736746383846 | 1\n", - "^3HN31E1... | 4 | 2 | 3 | 1736746383846 | 1\n" + "^YYY4HAB... | 1 | 5 | 6 | 1736832797128 | 1\n", + "^Z3QWT29... | 2 | 1 | 2 | 1736832797128 | 1\n", + "^3CZ78B4... | 3 | 1 | 2 | 1736832797128 | 1\n", + "^3HN31E1... | 4 | 3 | 4 | 1736832797128 | 1\n", + "^YYY4HAB... | 1 | 5 | 6 | 1736832797628 | -1\n", + "^Z3QWT29... | 2 | 1 | 2 | 1736832797628 | -1\n", + "^3CZ78B4... | 3 | 1 | 2 | 1736832797628 | -1\n", + "^3HN31E1... | 4 | 3 | 4 | 1736832797628 | -1\n", + "^YYY4HAB... | 1 | 2 | 3 | 1736832797628 | 1\n", + "^Z3QWT29... | 2 | 4 | 5 | 1736832797628 | 1\n", + "^3CZ78B4... | 3 | 2 | 3 | 1736832797628 | 1\n", + "^3HN31E1... | 4 | 2 | 3 | 1736832797628 | 1\n" ] } ], @@ -688,16 +688,16 @@ "output_type": "stream", "text": [ " | group | value | ret | __time__ | __diff__\n", - "^Z3QWT29... | 2 | 1 | 2 | 1736746383982 | 1\n", - "^3HN31E1... | 4 | 3 | 4 | 1736746383982 | 1\n", - "^Z3QWT29... | 2 | 1 | 2 | 1736746384282 | -1\n", - "^3HN31E1... | 4 | 3 | 4 | 1736746384282 | -1\n", - "^YYY4HAB... | 1 | 5 | 6 | 1736746384382 | 1\n", - "^3CZ78B4... | 3 | 1 | 2 | 1736746384382 | 1\n", - "^YYY4HAB... | 1 | 5 | 6 | 1736746384482 | -1\n", - "^3CZ78B4... | 3 | 1 | 2 | 1736746384482 | -1\n", - "^YYY4HAB... | 1 | 2 | 3 | 1736746384482 | 1\n", - "^3CZ78B4... | 3 | 2 | 3 | 1736746384482 | 1\n" + "^Z3QWT29... | 2 | 1 | 2 | 1736832797816 | 1\n", + "^3HN31E1... | 4 | 3 | 4 | 1736832797816 | 1\n", + "^Z3QWT29... | 2 | 1 | 2 | 1736832798116 | -1\n", + "^3HN31E1... | 4 | 3 | 4 | 1736832798116 | -1\n", + "^YYY4HAB... | 1 | 5 | 6 | 1736832798218 | 1\n", + "^3CZ78B4... | 3 | 1 | 2 | 1736832798218 | 1\n", + "^YYY4HAB... | 1 | 5 | 6 | 1736832798316 | -1\n", + "^3CZ78B4... | 3 | 1 | 2 | 1736832798316 | -1\n", + "^YYY4HAB... | 1 | 2 | 3 | 1736832798316 | 1\n", + "^3CZ78B4... | 3 | 2 | 3 | 1736832798316 | 1\n" ] } ], diff --git a/examples/notebooks/tutorials/iterate.ipynb b/examples/notebooks/tutorials/iterate.ipynb index b8cc3f1a..74f5f05c 100644 --- a/examples/notebooks/tutorials/iterate.ipynb +++ b/examples/notebooks/tutorials/iterate.ipynb @@ -370,7 +370,7 @@ "metadata": {}, "source": [ "\n", - "In an iteration step, the `edges` table is joined with the `vertices` table to get the representatives of neighbors in the graph. Then `groupby` is performed on `edges_with_repr` to get a minimal representative for each vertex. A new ID is assigned based on column `a` - vertex label. It is assigned in exactly the same way it is done above when creating a table. It allows you to have the same set of keys in the `vertices_updated` table as in the `vertices` table. However, Pathway is not that clever to deduce that the keys are exactly the same in these two tables. That's why it has to be additionaly told they are the same, by using [`with_universe_of`](/developers/api-docs/pathway-table#pathway.Table.with_universe_of).\n", + "In an iteration step, the `edges` table is joined with the `vertices` table to get the representatives of neighbors in the graph. Then `groupby` is performed on `edges_with_repr` to get a minimal representative for each vertex. A new ID is assigned based on column `a` - vertex label. It is assigned in exactly the same way it is done above when creating a table. It allows you to have the same set of keys in the `vertices_updated` table as in the `vertices` table. However, Pathway is not that clever to deduce that the keys are exactly the same in these two tables. That's why it has to be additionally told they are the same, by using [`with_universe_of`](/developers/api-docs/pathway-table#pathway.Table.with_universe_of).\n", "\n", "Preserving the set of keys is important in `iterate`. The iteration can stop only stop if there are no updates in any of the records. The records correspondence between iterations is determined using their IDs. If a record with one ID disappears and a record with a new ID appears, Pathway decides that something is still changing and the computation has to continue (even if the contents of the two rows are the same). It is possible to change the set of keys used in `iterate` but in the end the set of keys has to stop changing anyway. You can see that in the next example on computing shortest distances in a graph." ] diff --git a/examples/notebooks/tutorials/option-greeks.ipynb b/examples/notebooks/tutorials/option-greeks.ipynb index c68abb4c..04d5738b 100644 --- a/examples/notebooks/tutorials/option-greeks.ipynb +++ b/examples/notebooks/tutorials/option-greeks.ipynb @@ -758,7 +758,8 @@ "name": "stdout", "output_type": "stream", "text": [ - " | ts_recv | raw_symbol | expiration | instrument_class | strike_price | underlying | instrument_id | option_midprice | future_price\n", + " | ts_recv | raw_symbol | expiration | instrument_class | strike_price | underlying | instrument_id | option_midprice | future_price", + "\n", "^6ZHYYVZ... | 1712275200000000000 | ESM4 C3950 | 1718976600000000000 | C | 3950.0 | ESM4 | 4162742 | 1255.875 | 5203.894756928135\n", "^W64CERT... | 1712275200000000000 | ESM4 C4000 | 1718976600000000000 | C | 4000.0 | ESM4 | 574893 | 1206.25 | 5203.894756928135\n", "^WGHCM9D... | 1712275200000000000 | ESM4 C4050 | 1718976600000000000 | C | 4050.0 | ESM4 | 4162333 | 1156.75 | 5203.894756928135\n",