From fc886a75940771c85bc89d54f40d4377160dad16 Mon Sep 17 00:00:00 2001
From: Piotr Mardziel <piotrm@gmail.com>
Date: Tue, 30 Apr 2024 18:59:09 -0700
Subject: [PATCH 1/4] working on json to OT value mapping

---
 .../examples/experimental/spans_example.ipynb | 1139 ++++-------------
 trulens_eval/trulens_eval/trace/span.py       |    1 +
 2 files changed, 236 insertions(+), 904 deletions(-)

diff --git a/trulens_eval/examples/experimental/spans_example.ipynb b/trulens_eval/examples/experimental/spans_example.ipynb
index bbe253967..5450f9cb0 100644
--- a/trulens_eval/examples/experimental/spans_example.ipynb
+++ b/trulens_eval/examples/experimental/spans_example.ipynb
@@ -1,5 +1,224 @@
 {
  "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def lense_json(\n",
+    "    val: TPyJSON, lens: Lens = None\n",
+    ") -> Dict[Lens, ot_types.AttributeValue]:\n",
+    "    \"\"\"Convert a json structure to an OpenTelemetry attributes dict.\n",
+    "\n",
+    "    OT dicts have limited types in their values so a single JSON may need to be\n",
+    "    converted to multiple OT key/value pairs. The approach is to store\n",
+    "    paths/lenses into basic JSON elements, serialized into strings, as keys and\n",
+    "    those basic elements as values.\n",
+    "    \"\"\"\n",
+    "\n",
+    "    if lens is None:\n",
+    "        lens = Lens()\n",
+    "\n",
+    "    if isinstance(val, TPyJSONBase):\n",
+    "        return {lens: val}\n",
+    "\n",
+    "    if isinstance(val, Dict):\n",
+    "        ret = {}\n",
+    "        for k, v in val.items():\n",
+    "            sublens = lens[k]\n",
+    "            ret.update(lense_json(v, sublens))\n",
+    "        return ret\n",
+    "\n",
+    "    elif isinstance(val, Sequence):\n",
+    "        if len(val) == 0:\n",
+    "                return {lens: val}\n",
+    "        typ = type(val[0])\n",
+    "\n",
+    "        if isinstance(val[0], TPyJSON) and all(isinstance(x, typ) for x in val):\n",
+    "            return {lens: val}\n",
+    "        \n",
+    "        ret = {}\n",
+    "        for i in range(len(val)):\n",
+    "            sublens = lens[i]\n",
+    "            subval = val[i]\n",
+    "            ret.update(lense_json(subval, sublens))\n",
+    "\n",
+    "        return ret\n",
+    "\n",
+    "    else:\n",
+    "        raise ValueError(f\"Unexpected type: {type(val)}\")\n",
+    "\n",
+    "lense_json({\"hello\": 1, \"true\": 2})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import Union, Dict, List, Mapping, Sequence, TypeVar, Generic, Optional\n",
+    "\n",
+    "TPyJSONBase = Union[str, int, float, bool]\n",
+    "\n",
+    "TPyJSON = Union[TPyJSONBase, Mapping[str, 'TPyJSON'], Sequence['TPyJSON']]\n",
+    "from opentelemetry.util import types as ot_types\n",
+    "\n",
+    "\"\"\"\n",
+    "AttributeValue = Union[\n",
+    "    str,\n",
+    "    bool,\n",
+    "    int,\n",
+    "    float,\n",
+    "    Sequence[str],\n",
+    "    Sequence[bool],\n",
+    "    Sequence[int],\n",
+    "    Sequence[float],\n",
+    "]\n",
+    "\"\"\"\n",
+    "\n",
+    "from trulens_eval.utils.serial import Lens\n",
+    "\n",
+    "T = TypeVar(\"T\")\n",
+    "\n",
+    "class LensMappedDict(Generic[T], Dict[str, T]):\n",
+    "    def __init__(self, store: Optional[dict] = None, lens: Lens = None):\n",
+    "        if store is None:\n",
+    "            store = {}\n",
+    "        self.store = store\n",
+    "\n",
+    "        if lens is None:\n",
+    "            lens = Lens() \n",
+    "        self.lens = lens\n",
+    "\n",
+    "        self.relative_store = {}\n",
+    "\n",
+    "    def __repr__(self):\n",
+    "        return repr(self.relative_store)\n",
+    "\n",
+    "    def __str__(self):\n",
+    "        return str(self.relative_store)\n",
+    "\n",
+    "    def unmap_value(self, value: Union[T, 'LensMappedDict[T]']) -> T:\n",
+    "        if isinstance(value, LensMappedDict):\n",
+    "            ret = {}\n",
+    "            for k, v in value.relative_store.items():\n",
+    "                ret[k] = value.unmap_value(v)\n",
+    "\n",
+    "            return ret\n",
+    "            \n",
+    "        elif isinstance(value, TPyJSONBase):\n",
+    "            return value\n",
+    "\n",
+    "        elif value is None:\n",
+    "            return value\n",
+    "\n",
+    "        else:\n",
+    "            raise ValueError(f\"Unexpected type: {type(value)}\")\n",
+    "\n",
+    "    def map_value(self, lens: Lens, value: T) -> Union[T, 'LensMappedDict[T]']:\n",
+    "        if isinstance(value, TPyJSONBase):\n",
+    "            return value\n",
+    "\n",
+    "        elif isinstance(value, Dict):\n",
+    "            temp = LensMappedDict(store=self.store, lens=lens)\n",
+    "            for k, v in value.items():\n",
+    "                temp[k] = v\n",
+    "            return temp\n",
+    "\n",
+    "        elif isinstance(value, Sequence):\n",
+    "            if len(value) == 0:\n",
+    "                return value\n",
+    "            typ = type(value[0])\n",
+    "\n",
+    "            if isinstance(value[0], TPyJSON) and all(isinstance(x, typ) for x in value):\n",
+    "                return value\n",
+    "\n",
+    "            temp = LensMappedDict(store=self.store, lens=lens)\n",
+    "            for i in range(len(value)):\n",
+    "                temp[i] = value[i]\n",
+    "\n",
+    "            return temp\n",
+    "\n",
+    "        else:\n",
+    "            raise ValueError(f\"Unexpected type: {type(value)}\")\n",
+    "\n",
+    "    def __getitem__(self, key: str, default: Optional[T] = None) -> T:\n",
+    "        return self.relative_store.get(key, default)\n",
+    "\n",
+    "    def __setitem__(self, key: str, value: T) -> None:\n",
+    "        lens = self.lens[key]\n",
+    "        mapped_val = self.map_value(lens, value)\n",
+    "\n",
+    "        if key in self.relative_store:\n",
+    "            # Need to delete as there might be multiple store lenses associated\n",
+    "            # with this key.\n",
+    "            del self[key]\n",
+    "        \n",
+    "        self.store[lens] = mapped_val\n",
+    "        self.relative_store[key] = mapped_val\n",
+    "\n",
+    "    def __delitem__(self, key: str) -> None:\n",
+    "        lens = self.lens[key]\n",
+    "        val = self.relative_store[key]\n",
+    "\n",
+    "        if isinstance(val, LensMappedDict):\n",
+    "            del val\n",
+    "\n",
+    "        del self.store[lens]\n",
+    "        del self.relative_store[key]\n",
+    "\n",
+    "    def __del__(self):\n",
+    "        for k, v in list(self.relative_store.items()):\n",
+    "            sublens = self.lens[k]\n",
+    "\n",
+    "            if hasattr(v, \"__del__\"):\n",
+    "                del v\n",
+    "\n",
+    "            del self.relative_store[k]\n",
+    "            del self.store[sublens]\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "temp = LensMappedDict()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "temp['something'] = {'a': 1, 'b': 2, 'sub': {'c': 3, 'd': 4}}\n",
+    "temp.store"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "temp['something']['sub'] = 42\n",
+    "temp.store"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "type(temp.relative_store['something'])"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -9,7 +228,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -25,7 +244,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -43,53 +262,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "🦑 Tru initialized with db url sqlite:///default.sqlite .\n",
-      "🛑 Secret keys may be written to the database. See the `database_redact_keys` option of Tru` to prevent this.\n",
-      "Force stopping dashboard ...\n",
-      "Starting dashboard ...\n",
-      "Config file already exists. Skipping writing process.\n",
-      "Credentials file already exists. Skipping writing process.\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "0291bf2072e44da99976977bb060cd76",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Accordion(children=(VBox(children=(VBox(children=(Label(value='STDOUT'), Output())), VBox(children=(Label(valu…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Dashboard started at http://10.0.0.25:8501 .\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "<Popen: returncode: None args: ['streamlit', 'run', '--server.headless=True'...>"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "from trulens_eval import Tru\n",
     "Tru().reset_database()\n",
@@ -98,7 +273,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -114,116 +289,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Module examples.expositional.end2end_apps.custom_app.custom_app*\n",
-      "  Class examples.expositional.end2end_apps.custom_app.custom_app.CustomApp\n",
-      "    Method get_context: (self, input: str)\n",
-      "    Method respond_to_query: (self, input: str)\n",
-      "    Method arespond_to_query: (self, input: str)\n",
-      "  Class examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate\n",
-      "    Method fill: (self, question, answer)\n",
-      "\n",
-      "Module examples.expositional.end2end_apps.custom_app.custom_llm*\n",
-      "  Class examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM\n",
-      "    Method generate: (self, prompt: str, temperature: Optional[float] = None) -> str\n",
-      "      Span type: SpanType.LLM (static), spanner: <function CustomLLM.set_llm_span at 0x2abacc360>\n",
-      "      Span type: SpanType.LLM (id: 11463407760=?11463407760), spanner: <function CustomLLM.set_llm_span at 0x2abacc360>\n",
-      "\n",
-      "Module examples.expositional.end2end_apps.custom_app.custom_memory*\n",
-      "  Class examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory\n",
-      "    Method remember: (self, data: str)\n",
-      "      Span type: SpanType.MEMORY (static), spanner: <function CustomMemory.set_memory_span at 0x2abacc860>\n",
-      "      Span type: SpanType.MEMORY (id: 11474988304=?11474988304), spanner: <function CustomMemory.set_memory_span at 0x2abacc860>\n",
-      "\n",
-      "Module examples.expositional.end2end_apps.custom_app.custom_reranker*\n",
-      "  Class examples.expositional.end2end_apps.custom_app.custom_reranker.CustomReranker\n",
-      "    Method rerank: (self, query_text: str, chunks: List[str], chunk_scores: Optional[List[float]] = None) -> List[Tuple[str, float]]\n",
-      "      Span type: SpanType.RERANKER (static), spanner: <function CustomReranker.update_span at 0x2abacccc0>\n",
-      "      Span type: SpanType.RERANKER (id: 11475027856=?11475027856), spanner: <function CustomReranker.update_span at 0x2abacccc0>\n",
-      "\n",
-      "Module examples.expositional.end2end_apps.custom_app.custom_retriever*\n",
-      "  Class examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever\n",
-      "    Method retrieve_chunks: (self, data)\n",
-      "      Span type: SpanType.RETRIEVER (static), spanner: <function CustomRetriever.update_span at 0x2abacd1c0>\n",
-      "      Span type: SpanType.RETRIEVER (id: 11474992912=?11474992912), spanner: <function CustomRetriever.update_span at 0x2abacd1c0>\n",
-      "\n",
-      "Module examples.expositional.end2end_apps.custom_app.custom_tool*\n",
-      "  Class examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool\n",
-      "    Method invoke: (self, data: str)\n",
-      "      Span type: SpanType.TOOL (static), spanner: <function CustomTool.set_memory_span at 0x2abacd620>\n",
-      "      Span type: SpanType.TOOL (id: 11472633296=?11472633296), spanner: <function CustomTool.set_memory_span at 0x2abacd620>\n",
-      "      Span type: SpanType.TOOL (id: 4426749456=?4426749456), spanner: <function CustomTool.set_memory_span at 0x2abacd620>\n",
-      "      Span type: SpanType.TOOL (id: 11475026896=?11475026896), spanner: <function CustomTool.set_memory_span at 0x2abacd620>\n",
-      "\n",
-      "Module trulens_eval.*\n",
-      "  Class trulens_eval.feedback.feedback.Feedback\n",
-      "    Method __call__: (self, *args, **kwargs) -> 'Any'\n",
-      "\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "instruments.Instrument().print_instrumentation()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Components:\n",
-      "\tTruCustomApp (Other) at 0x2abe93ac0 with path __app__\n",
-      "\tCustomApp (Custom) at 0x2aba99490 with path __app__.app\n",
-      "\tCustomLLM (Custom) at 0x2ab45b890 with path __app__.app.llm\n",
-      "\tCustomMemory (Custom) at 0x2abf66d10 with path __app__.app.memory\n",
-      "\tCustomReranker (Custom) at 0x2abf70790 with path __app__.app.reranker\n",
-      "\tCustomRetriever (Custom) at 0x2abf67f10 with path __app__.app.retriever\n",
-      "\tCustomTemplate (Custom) at 0x2abf66750 with path __app__.app.template\n",
-      "\tCustomTool (Custom) at 0x2abd27dd0 with path __app__.app.tools[0]\n",
-      "\tCustomTool (Custom) at 0x107dad610 with path __app__.app.tools[1]\n",
-      "\tCustomTool (Custom) at 0x2abf703d0 with path __app__.app.tools[2]\n",
-      "\n",
-      "Methods:\n",
-      "Object at 0x2abf66d10:\n",
-      "\t<function CustomMemory.remember at 0x2abacc720> with path __app__.app.memory\n",
-      "Object at 0x2abf67f10:\n",
-      "\t<function CustomRetriever.retrieve_chunks at 0x2abacd080> with path __app__.app.retriever\n",
-      "Object at 0x2ab45b890:\n",
-      "\t<function CustomLLM.generate at 0x107dc0cc0> with path __app__.app.llm\n",
-      "Object at 0x2abf66750:\n",
-      "\t<function CustomTemplate.fill at 0x2abacd3a0> with path __app__.app.template\n",
-      "Object at 0x2abf70790:\n",
-      "\t<function CustomReranker.rerank at 0x2abaccb80> with path __app__.app.reranker\n",
-      "Object at 0x2aba99490:\n",
-      "\t<function CustomApp.get_context at 0x2abacd800> with path __app__.app\n",
-      "\t<function CustomApp.respond_to_query at 0x2abacd8a0> with path __app__.app\n",
-      "\t<function CustomApp.arespond_to_query at 0x2abacd940> with path __app__.app\n",
-      "Object at 0x2abd27dd0:\n",
-      "\t<function CustomTool.invoke at 0x2abacd4e0> with path __app__.app.tools[0]\n",
-      "Object at 0x107dad610:\n",
-      "\t<function CustomTool.invoke at 0x2abacd4e0> with path __app__.app.tools[1]\n",
-      "Object at 0x2abf703d0:\n",
-      "\t<function CustomTool.invoke at 0x2abacd4e0> with path __app__.app.tools[2]\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "ta.print_instrumented()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -235,250 +319,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'stack': [{'path': 'app',\n",
-       "   'method': {'obj': {'cls': {'name': 'CustomApp',\n",
-       "      'module': {'package_name': 'examples.expositional.end2end_apps.custom_app',\n",
-       "       'module_name': 'examples.expositional.end2end_apps.custom_app.custom_app'},\n",
-       "      'bases': None},\n",
-       "     'id': 11469952144,\n",
-       "     'init_bindings': None},\n",
-       "    'name': 'respond_to_query'}},\n",
-       "  {'path': 'app',\n",
-       "   'method': {'obj': {'cls': {'name': 'CustomApp',\n",
-       "      'module': {'package_name': 'examples.expositional.end2end_apps.custom_app',\n",
-       "       'module_name': 'examples.expositional.end2end_apps.custom_app.custom_app'},\n",
-       "      'bases': None},\n",
-       "     'id': 11469952144,\n",
-       "     'init_bindings': None},\n",
-       "    'name': 'get_context'}},\n",
-       "  {'path': 'app.retriever',\n",
-       "   'method': {'obj': {'cls': {'name': 'CustomRetriever',\n",
-       "      'module': {'package_name': 'examples.expositional.end2end_apps.custom_app',\n",
-       "       'module_name': 'examples.expositional.end2end_apps.custom_app.custom_retriever'},\n",
-       "      'bases': None},\n",
-       "     'id': 11474992912,\n",
-       "     'init_bindings': None},\n",
-       "    'name': 'retrieve_chunks'}}],\n",
-       " 'args': {'data': 'hello'},\n",
-       " 'rets': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'],\n",
-       " 'error': None,\n",
-       " 'perf': {'start_time': '2024-04-29T18:59:05.773577',\n",
-       "  'end_time': '2024-04-29T18:59:05.802310'},\n",
-       " 'pid': 41174,\n",
-       " 'tid': 15934086}"
-      ]
-     },
-     "execution_count": 8,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "rec.calls[0].model_dump()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>trace_id</th>\n",
-       "      <th>name</th>\n",
-       "      <th>type</th>\n",
-       "      <th>span_type</th>\n",
-       "      <th>span_id</th>\n",
-       "      <th>parent_span_id</th>\n",
-       "      <th>attributes</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>121</td>\n",
-       "      <td>root</td>\n",
-       "      <td>&lt;class 'trulens_eval.trace.span.SpanRoot'&gt;</td>\n",
-       "      <td>SpanRoot</td>\n",
-       "      <td>204</td>\n",
-       "      <td>0</td>\n",
-       "      <td>{'trulens_eval@span_type': 'SpanRoot', 'trulen...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>121</td>\n",
-       "      <td>retrieve_chunks</td>\n",
-       "      <td>&lt;class 'trulens_eval.trace.span.SpanRetriever'&gt;</td>\n",
-       "      <td>SpanRetriever</td>\n",
-       "      <td>56</td>\n",
-       "      <td>179</td>\n",
-       "      <td>{'trulens_eval@span_type': 'SpanRetriever', 't...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>121</td>\n",
-       "      <td>rerank</td>\n",
-       "      <td>&lt;class 'trulens_eval.trace.span.SpanReranker'&gt;</td>\n",
-       "      <td>SpanReranker</td>\n",
-       "      <td>197</td>\n",
-       "      <td>179</td>\n",
-       "      <td>{'trulens_eval@span_type': 'SpanReranker', 'tr...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>121</td>\n",
-       "      <td>invoke</td>\n",
-       "      <td>&lt;class 'trulens_eval.trace.span.SpanTool'&gt;</td>\n",
-       "      <td>SpanTool</td>\n",
-       "      <td>214</td>\n",
-       "      <td>179</td>\n",
-       "      <td>{'trulens_eval@span_type': 'SpanTool', 'trulen...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>121</td>\n",
-       "      <td>invoke</td>\n",
-       "      <td>&lt;class 'trulens_eval.trace.span.SpanTool'&gt;</td>\n",
-       "      <td>SpanTool</td>\n",
-       "      <td>40</td>\n",
-       "      <td>179</td>\n",
-       "      <td>{'trulens_eval@span_type': 'SpanTool', 'trulen...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5</th>\n",
-       "      <td>121</td>\n",
-       "      <td>get_context</td>\n",
-       "      <td>&lt;class 'trulens_eval.trace.span.SpanOther'&gt;</td>\n",
-       "      <td>SpanOther</td>\n",
-       "      <td>179</td>\n",
-       "      <td>14</td>\n",
-       "      <td>{'trulens_eval@span_type': 'SpanOther', 'trule...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>6</th>\n",
-       "      <td>121</td>\n",
-       "      <td>remember</td>\n",
-       "      <td>&lt;class 'trulens_eval.trace.span.SpanMemory'&gt;</td>\n",
-       "      <td>SpanMemory</td>\n",
-       "      <td>184</td>\n",
-       "      <td>14</td>\n",
-       "      <td>{'trulens_eval@span_type': 'SpanMemory', 'trul...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>7</th>\n",
-       "      <td>121</td>\n",
-       "      <td>generate</td>\n",
-       "      <td>&lt;class 'trulens_eval.trace.span.SpanLLM'&gt;</td>\n",
-       "      <td>SpanLLM</td>\n",
-       "      <td>78</td>\n",
-       "      <td>14</td>\n",
-       "      <td>{'trulens_eval@span_type': 'SpanLLM', 'trulens...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>8</th>\n",
-       "      <td>121</td>\n",
-       "      <td>fill</td>\n",
-       "      <td>&lt;class 'trulens_eval.trace.span.SpanOther'&gt;</td>\n",
-       "      <td>SpanOther</td>\n",
-       "      <td>134</td>\n",
-       "      <td>14</td>\n",
-       "      <td>{'trulens_eval@span_type': 'SpanOther', 'trule...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9</th>\n",
-       "      <td>121</td>\n",
-       "      <td>remember</td>\n",
-       "      <td>&lt;class 'trulens_eval.trace.span.SpanMemory'&gt;</td>\n",
-       "      <td>SpanMemory</td>\n",
-       "      <td>85</td>\n",
-       "      <td>14</td>\n",
-       "      <td>{'trulens_eval@span_type': 'SpanMemory', 'trul...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>10</th>\n",
-       "      <td>121</td>\n",
-       "      <td>respond_to_query</td>\n",
-       "      <td>&lt;class 'trulens_eval.trace.span.SpanOther'&gt;</td>\n",
-       "      <td>SpanOther</td>\n",
-       "      <td>14</td>\n",
-       "      <td>204</td>\n",
-       "      <td>{'trulens_eval@span_type': 'SpanOther', 'trule...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "    trace_id              name  \\\n",
-       "0        121              root   \n",
-       "1        121   retrieve_chunks   \n",
-       "2        121            rerank   \n",
-       "3        121            invoke   \n",
-       "4        121            invoke   \n",
-       "5        121       get_context   \n",
-       "6        121          remember   \n",
-       "7        121          generate   \n",
-       "8        121              fill   \n",
-       "9        121          remember   \n",
-       "10       121  respond_to_query   \n",
-       "\n",
-       "                                               type      span_type  span_id  \\\n",
-       "0        <class 'trulens_eval.trace.span.SpanRoot'>       SpanRoot      204   \n",
-       "1   <class 'trulens_eval.trace.span.SpanRetriever'>  SpanRetriever       56   \n",
-       "2    <class 'trulens_eval.trace.span.SpanReranker'>   SpanReranker      197   \n",
-       "3        <class 'trulens_eval.trace.span.SpanTool'>       SpanTool      214   \n",
-       "4        <class 'trulens_eval.trace.span.SpanTool'>       SpanTool       40   \n",
-       "5       <class 'trulens_eval.trace.span.SpanOther'>      SpanOther      179   \n",
-       "6      <class 'trulens_eval.trace.span.SpanMemory'>     SpanMemory      184   \n",
-       "7         <class 'trulens_eval.trace.span.SpanLLM'>        SpanLLM       78   \n",
-       "8       <class 'trulens_eval.trace.span.SpanOther'>      SpanOther      134   \n",
-       "9      <class 'trulens_eval.trace.span.SpanMemory'>     SpanMemory       85   \n",
-       "10      <class 'trulens_eval.trace.span.SpanOther'>      SpanOther       14   \n",
-       "\n",
-       "    parent_span_id                                         attributes  \n",
-       "0                0  {'trulens_eval@span_type': 'SpanRoot', 'trulen...  \n",
-       "1              179  {'trulens_eval@span_type': 'SpanRetriever', 't...  \n",
-       "2              179  {'trulens_eval@span_type': 'SpanReranker', 'tr...  \n",
-       "3              179  {'trulens_eval@span_type': 'SpanTool', 'trulen...  \n",
-       "4              179  {'trulens_eval@span_type': 'SpanTool', 'trulen...  \n",
-       "5               14  {'trulens_eval@span_type': 'SpanOther', 'trule...  \n",
-       "6               14  {'trulens_eval@span_type': 'SpanMemory', 'trul...  \n",
-       "7               14  {'trulens_eval@span_type': 'SpanLLM', 'trulens...  \n",
-       "8               14  {'trulens_eval@span_type': 'SpanOther', 'trule...  \n",
-       "9               14  {'trulens_eval@span_type': 'SpanMemory', 'trul...  \n",
-       "10             204  {'trulens_eval@span_type': 'SpanOther', 'trule...  "
-      ]
-     },
-     "execution_count": 9,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "spans = Categorizer.spans_of_record(rec)\n",
     "\n",
@@ -506,530 +358,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "SpanRoot(name='root', kind=<SpanKind.INTERNAL: 0>, status=<StatusCode.OK: 1>, status_description=None, start_timestamp=1714442346605224000, end_timestamp=1714417146452397000, context=HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0xc461bfb604e9f8cc, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={}, attributes={'trulens_eval@span_type': 'SpanRoot', 'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c'}, attributes_metadata={}, record=Record(record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), ts=datetime.datetime(2024, 4, 29, 18, 59, 6, 452457), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11474992912, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 773577), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 802310)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.reranker, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_reranker.CustomReranker, id=11475027856, init_bindings=None), name='rerank'))], args={'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, rets=[['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 929746), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 975183)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[1], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: HELLO'}, rets='RELEVANT CHUNK: HELLO', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 23272), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 38280)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[0], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: olleholleholleh'}, rets='relevant chunk: olleholleholleh', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 57370), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64115)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context'))], args={'input': 'hello'}, rets=['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 694946), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64200)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 128985), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 154244)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11463407760, init_bindings=None), name='generate'))], args={'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, rets=\"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 223581), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 249975)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11474986832, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 311945), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 335659)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 426073), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452334)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), pid=41174, tid=15934086)], feedback_and_future_results=[], feedback_results=[]), tags=[], span_type='SpanRoot', record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c')\n",
-      "{'attributes': {'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      "                'trulens_eval@span_type': 'SpanRoot'},\n",
-      " 'attributes_metadata': {},\n",
-      " 'context': (4444476882659008121, 14150802292662204620, False, 0, [], True),\n",
-      " 'end_timestamp': 1714417146452397000,\n",
-      " 'events': [],\n",
-      " 'kind': <SpanKind.INTERNAL: 0>,\n",
-      " 'links': [],\n",
-      " 'name': 'root',\n",
-      " 'record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      " 'span_type': 'SpanRoot',\n",
-      " 'start_timestamp': 1714442346605224000,\n",
-      " 'status': <StatusCode.OK: 1>,\n",
-      " 'status_description': None,\n",
-      " 'tags': []}\n",
-      "\n",
-      "SpanRetriever(name='retrieve_chunks', kind=<SpanKind.INTERNAL: 0>, status=<StatusCode.OK: 1>, status_description=None, start_timestamp=1714417145773577000, end_timestamp=1714417145802310000, context=HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x40edcfbc5dd89e38, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x9dfa7a73d5cb8fb3, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanRetriever', 'trulens_eval@num_contexts': 2, 'trulens_eval@query_text': 'hello', 'trulens_eval@query_embedding': [0.1444935915101181, 0.40050466892122316, 0.5538924770694639, 0.885428542923527, 0.9699794878996718, 0.8609791164486051, 0.08076587399485369, 0.5249545160411511, 0.6895703624502038, 0.3429581189302343, 0.09855133701627428, 0.6090099106242878, 0.475476525604301, 0.38980813300423256, 0.1512859413369846, 0.226748800479444], 'trulens_eval@distance_type': 'cosine-dummy-version', 'trulens_eval@retrieved_contexts': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'trulens_eval@retrieved_scores': [0.6587668707677713, 0.0745463501887782], 'trulens_eval@retrieved_embeddings': [[0.3764704966828506, 0.11768612290267688, 0.3156195870081395, 0.5851030853065906, 0.6370756233058007, 0.19646488567493203, 0.4351959030860185, 0.006515799863664462, 0.8836303135178527, 0.1432266410573454, 0.7961021901832533, 0.3723581115611062, 0.6686891235803216, 0.09788058397125066, 0.7368721257131602, 0.4593322183656744], [0.1476617705133907, 0.7650886751115091, 0.672538379808745, 0.32097067821776326, 0.11074541968997242, 0.9372453033369343, 0.7036299923932721, 0.208622667779918, 0.9933605724200493, 0.40281198841660215, 0.1828814633365755, 0.781004025469402, 0.5066063631033478, 0.3123927125352196, 0.9909259356434352, 0.5680672550338478]], 'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c', 'trulens_eval@inputs': {'data': 'hello'}, 'trulens_eval@output': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), ts=datetime.datetime(2024, 4, 29, 18, 59, 6, 452457), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11474992912, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 773577), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 802310)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.reranker, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_reranker.CustomReranker, id=11475027856, init_bindings=None), name='rerank'))], args={'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, rets=[['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 929746), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 975183)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[1], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: HELLO'}, rets='RELEVANT CHUNK: HELLO', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 23272), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 38280)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[0], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: olleholleholleh'}, rets='relevant chunk: olleholleholleh', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 57370), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64115)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context'))], args={'input': 'hello'}, rets=['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 694946), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64200)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 128985), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 154244)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11463407760, init_bindings=None), name='generate'))], args={'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, rets=\"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 223581), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 249975)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11474986832, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 311945), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 335659)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 426073), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452334)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), pid=41174, tid=15934086)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11474992912, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 773577), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 802310)), pid=41174, tid=15934086), tags=[], span_type='SpanRetriever', record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', inputs={'data': 'hello'}, output=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], error=None, query_text='hello', query_embedding=[0.1444935915101181, 0.40050466892122316, 0.5538924770694639, 0.885428542923527, 0.9699794878996718, 0.8609791164486051, 0.08076587399485369, 0.5249545160411511, 0.6895703624502038, 0.3429581189302343, 0.09855133701627428, 0.6090099106242878, 0.475476525604301, 0.38980813300423256, 0.1512859413369846, 0.226748800479444], distance_type='cosine-dummy-version', num_contexts=2, retrieved_contexts=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], retrieved_scores=[0.6587668707677713, 0.0745463501887782], retrieved_embeddings=[[0.3764704966828506, 0.11768612290267688, 0.3156195870081395, 0.5851030853065906, 0.6370756233058007, 0.19646488567493203, 0.4351959030860185, 0.006515799863664462, 0.8836303135178527, 0.1432266410573454, 0.7961021901832533, 0.3723581115611062, 0.6686891235803216, 0.09788058397125066, 0.7368721257131602, 0.4593322183656744], [0.1476617705133907, 0.7650886751115091, 0.672538379808745, 0.32097067821776326, 0.11074541968997242, 0.9372453033369343, 0.7036299923932721, 0.208622667779918, 0.9933605724200493, 0.40281198841660215, 0.1828814633365755, 0.781004025469402, 0.5066063631033478, 0.3123927125352196, 0.9909259356434352, 0.5680672550338478]])\n",
-      "{'attributes': {'trulens_eval@distance_type': 'cosine-dummy-version',\n",
-      "                'trulens_eval@error': None,\n",
-      "                'trulens_eval@inputs': {'data': 'hello'},\n",
-      "                'trulens_eval@num_contexts': 2,\n",
-      "                'trulens_eval@output': ['Relevant chunk: HELLO',\n",
-      "                                        'Relevant chunk: olleholleholleh'],\n",
-      "                'trulens_eval@query_embedding': [0.1444935915101181,\n",
-      "                                                 0.40050466892122316,\n",
-      "                                                 0.5538924770694639,\n",
-      "                                                 0.885428542923527,\n",
-      "                                                 0.9699794878996718,\n",
-      "                                                 0.8609791164486051,\n",
-      "                                                 0.08076587399485369,\n",
-      "                                                 0.5249545160411511,\n",
-      "                                                 0.6895703624502038,\n",
-      "                                                 0.3429581189302343,\n",
-      "                                                 0.09855133701627428,\n",
-      "                                                 0.6090099106242878,\n",
-      "                                                 0.475476525604301,\n",
-      "                                                 0.38980813300423256,\n",
-      "                                                 0.1512859413369846,\n",
-      "                                                 0.226748800479444],\n",
-      "                'trulens_eval@query_text': 'hello',\n",
-      "                'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      "                'trulens_eval@retrieved_contexts': ['Relevant chunk: HELLO',\n",
-      "                                                    'Relevant chunk: '\n",
-      "                                                    'olleholleholleh'],\n",
-      "                'trulens_eval@retrieved_embeddings': [[0.3764704966828506,\n",
-      "                                                       0.11768612290267688,\n",
-      "                                                       0.3156195870081395,\n",
-      "                                                       0.5851030853065906,\n",
-      "                                                       0.6370756233058007,\n",
-      "                                                       0.19646488567493203,\n",
-      "                                                       0.4351959030860185,\n",
-      "                                                       0.006515799863664462,\n",
-      "                                                       0.8836303135178527,\n",
-      "                                                       0.1432266410573454,\n",
-      "                                                       0.7961021901832533,\n",
-      "                                                       0.3723581115611062,\n",
-      "                                                       0.6686891235803216,\n",
-      "                                                       0.09788058397125066,\n",
-      "                                                       0.7368721257131602,\n",
-      "                                                       0.4593322183656744],\n",
-      "                                                      [0.1476617705133907,\n",
-      "                                                       0.7650886751115091,\n",
-      "                                                       0.672538379808745,\n",
-      "                                                       0.32097067821776326,\n",
-      "                                                       0.11074541968997242,\n",
-      "                                                       0.9372453033369343,\n",
-      "                                                       0.7036299923932721,\n",
-      "                                                       0.208622667779918,\n",
-      "                                                       0.9933605724200493,\n",
-      "                                                       0.40281198841660215,\n",
-      "                                                       0.1828814633365755,\n",
-      "                                                       0.781004025469402,\n",
-      "                                                       0.5066063631033478,\n",
-      "                                                       0.3123927125352196,\n",
-      "                                                       0.9909259356434352,\n",
-      "                                                       0.5680672550338478]],\n",
-      "                'trulens_eval@retrieved_scores': [0.6587668707677713,\n",
-      "                                                  0.0745463501887782],\n",
-      "                'trulens_eval@span_type': 'SpanRetriever'},\n",
-      " 'attributes_metadata': {},\n",
-      " 'context': (4444476882659008121, 4678623995843092024, False, 0, [], True),\n",
-      " 'distance_type': 'cosine-dummy-version',\n",
-      " 'end_timestamp': 1714417145802310000,\n",
-      " 'error': None,\n",
-      " 'events': [],\n",
-      " 'inputs': {'data': 'hello'},\n",
-      " 'kind': <SpanKind.INTERNAL: 0>,\n",
-      " 'links': [((4444476882659008121, 11383545646059065267, False, 0, [], True),\n",
-      "            {'trulens_eval@relationship': 'parent'})],\n",
-      " 'name': 'retrieve_chunks',\n",
-      " 'num_contexts': 2,\n",
-      " 'output': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'],\n",
-      " 'query_embedding': [0.1444935915101181,\n",
-      "                     0.40050466892122316,\n",
-      "                     0.5538924770694639,\n",
-      "                     0.885428542923527,\n",
-      "                     0.9699794878996718,\n",
-      "                     0.8609791164486051,\n",
-      "                     0.08076587399485369,\n",
-      "                     0.5249545160411511,\n",
-      "                     0.6895703624502038,\n",
-      "                     0.3429581189302343,\n",
-      "                     0.09855133701627428,\n",
-      "                     0.6090099106242878,\n",
-      "                     0.475476525604301,\n",
-      "                     0.38980813300423256,\n",
-      "                     0.1512859413369846,\n",
-      "                     0.226748800479444],\n",
-      " 'query_text': 'hello',\n",
-      " 'record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      " 'retrieved_contexts': ['Relevant chunk: HELLO',\n",
-      "                        'Relevant chunk: olleholleholleh'],\n",
-      " 'retrieved_embeddings': [[0.3764704966828506,\n",
-      "                           0.11768612290267688,\n",
-      "                           0.3156195870081395,\n",
-      "                           0.5851030853065906,\n",
-      "                           0.6370756233058007,\n",
-      "                           0.19646488567493203,\n",
-      "                           0.4351959030860185,\n",
-      "                           0.006515799863664462,\n",
-      "                           0.8836303135178527,\n",
-      "                           0.1432266410573454,\n",
-      "                           0.7961021901832533,\n",
-      "                           0.3723581115611062,\n",
-      "                           0.6686891235803216,\n",
-      "                           0.09788058397125066,\n",
-      "                           0.7368721257131602,\n",
-      "                           0.4593322183656744],\n",
-      "                          [0.1476617705133907,\n",
-      "                           0.7650886751115091,\n",
-      "                           0.672538379808745,\n",
-      "                           0.32097067821776326,\n",
-      "                           0.11074541968997242,\n",
-      "                           0.9372453033369343,\n",
-      "                           0.7036299923932721,\n",
-      "                           0.208622667779918,\n",
-      "                           0.9933605724200493,\n",
-      "                           0.40281198841660215,\n",
-      "                           0.1828814633365755,\n",
-      "                           0.781004025469402,\n",
-      "                           0.5066063631033478,\n",
-      "                           0.3123927125352196,\n",
-      "                           0.9909259356434352,\n",
-      "                           0.5680672550338478]],\n",
-      " 'retrieved_scores': [0.6587668707677713, 0.0745463501887782],\n",
-      " 'span_type': 'SpanRetriever',\n",
-      " 'start_timestamp': 1714417145773577000,\n",
-      " 'status': <StatusCode.OK: 1>,\n",
-      " 'status_description': None,\n",
-      " 'tags': []}\n",
-      "\n",
-      "SpanReranker(name='rerank', kind=<SpanKind.INTERNAL: 0>, status=<StatusCode.OK: 1>, status_description=None, start_timestamp=1714417145929746000, end_timestamp=1714417145975183000, context=HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x3952719d56aa6dc5, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x9dfa7a73d5cb8fb3, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanReranker', 'trulens_eval@model_name': 'herpderp-v1-reranker', 'trulens_eval@top_n': 2, 'trulens_eval@query_text': 'hello', 'trulens_eval@input_context_texts': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'trulens_eval@input_score_scores': None, 'trulens_eval@output_ranks': [0, 1], 'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c', 'trulens_eval@inputs': {'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, 'trulens_eval@output': [['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), ts=datetime.datetime(2024, 4, 29, 18, 59, 6, 452457), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11474992912, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 773577), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 802310)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.reranker, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_reranker.CustomReranker, id=11475027856, init_bindings=None), name='rerank'))], args={'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, rets=[['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 929746), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 975183)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[1], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: HELLO'}, rets='RELEVANT CHUNK: HELLO', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 23272), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 38280)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[0], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: olleholleholleh'}, rets='relevant chunk: olleholleholleh', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 57370), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64115)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context'))], args={'input': 'hello'}, rets=['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 694946), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64200)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 128985), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 154244)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11463407760, init_bindings=None), name='generate'))], args={'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, rets=\"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 223581), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 249975)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11474986832, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 311945), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 335659)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 426073), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452334)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), pid=41174, tid=15934086)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.reranker, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_reranker.CustomReranker, id=11475027856, init_bindings=None), name='rerank'))], args={'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, rets=[['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 929746), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 975183)), pid=41174, tid=15934086), tags=[], span_type='SpanReranker', record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', inputs={'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, output=[['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], error=None, query_text='hello', model_name='herpderp-v1-reranker', top_n=2, input_context_texts=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], input_context_scores=None, output_ranks=[0, 1])\n",
-      "{'attributes': {'trulens_eval@error': None,\n",
-      "                'trulens_eval@input_context_texts': ['Relevant chunk: HELLO',\n",
-      "                                                     'Relevant chunk: '\n",
-      "                                                     'olleholleholleh'],\n",
-      "                'trulens_eval@input_score_scores': None,\n",
-      "                'trulens_eval@inputs': {'chunk_scores': None,\n",
-      "                                        'chunks': ['Relevant chunk: HELLO',\n",
-      "                                                   'Relevant chunk: '\n",
-      "                                                   'olleholleholleh'],\n",
-      "                                        'query_text': 'hello'},\n",
-      "                'trulens_eval@model_name': 'herpderp-v1-reranker',\n",
-      "                'trulens_eval@output': [['Relevant chunk: HELLO', 16.0],\n",
-      "                                        ['Relevant chunk: olleholleholleh',\n",
-      "                                         26.0]],\n",
-      "                'trulens_eval@output_ranks': [0, 1],\n",
-      "                'trulens_eval@query_text': 'hello',\n",
-      "                'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      "                'trulens_eval@span_type': 'SpanReranker',\n",
-      "                'trulens_eval@top_n': 2},\n",
-      " 'attributes_metadata': {},\n",
-      " 'context': (4444476882659008121, 4130488728829980101, False, 0, [], True),\n",
-      " 'end_timestamp': 1714417145975183000,\n",
-      " 'error': None,\n",
-      " 'events': [],\n",
-      " 'input_context_scores': None,\n",
-      " 'input_context_texts': ['Relevant chunk: HELLO',\n",
-      "                         'Relevant chunk: olleholleholleh'],\n",
-      " 'inputs': {'chunk_scores': None,\n",
-      "            'chunks': ['Relevant chunk: HELLO',\n",
-      "                       'Relevant chunk: olleholleholleh'],\n",
-      "            'query_text': 'hello'},\n",
-      " 'kind': <SpanKind.INTERNAL: 0>,\n",
-      " 'links': [((4444476882659008121, 11383545646059065267, False, 0, [], True),\n",
-      "            {'trulens_eval@relationship': 'parent'})],\n",
-      " 'model_name': 'herpderp-v1-reranker',\n",
-      " 'name': 'rerank',\n",
-      " 'output': [['Relevant chunk: HELLO', 16.0],\n",
-      "            ['Relevant chunk: olleholleholleh', 26.0]],\n",
-      " 'output_ranks': [0, 1],\n",
-      " 'query_text': 'hello',\n",
-      " 'record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      " 'span_type': 'SpanReranker',\n",
-      " 'start_timestamp': 1714417145929746000,\n",
-      " 'status': <StatusCode.OK: 1>,\n",
-      " 'status_description': None,\n",
-      " 'tags': [],\n",
-      " 'top_n': 2}\n",
-      "\n",
-      "SpanTool(name='invoke', kind=<SpanKind.INTERNAL: 0>, status=<StatusCode.OK: 1>, status_description=None, start_timestamp=1714417146023272000, end_timestamp=1714417146038280000, context=HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x2685ec5d82b23ad6, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x9dfa7a73d5cb8fb3, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanTool', 'trulens_eval@description': 'Return a copy of the string with trailing whitespace removed.\\n\\nIf chars is given and not None, remove characters in chars instead.', 'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c', 'trulens_eval@inputs': {'data': 'Relevant chunk: HELLO'}, 'trulens_eval@output': 'RELEVANT CHUNK: HELLO', 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), ts=datetime.datetime(2024, 4, 29, 18, 59, 6, 452457), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11474992912, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 773577), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 802310)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.reranker, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_reranker.CustomReranker, id=11475027856, init_bindings=None), name='rerank'))], args={'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, rets=[['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 929746), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 975183)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[1], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: HELLO'}, rets='RELEVANT CHUNK: HELLO', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 23272), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 38280)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[0], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: olleholleholleh'}, rets='relevant chunk: olleholleholleh', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 57370), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64115)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context'))], args={'input': 'hello'}, rets=['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 694946), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64200)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 128985), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 154244)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11463407760, init_bindings=None), name='generate'))], args={'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, rets=\"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 223581), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 249975)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11474986832, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 311945), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 335659)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 426073), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452334)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), pid=41174, tid=15934086)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[1], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: HELLO'}, rets='RELEVANT CHUNK: HELLO', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 23272), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 38280)), pid=41174, tid=15934405), tags=[], span_type='SpanTool', record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', inputs={'data': 'Relevant chunk: HELLO'}, output='RELEVANT CHUNK: HELLO', error=None, description='Return a copy of the string with trailing whitespace removed.\\n\\nIf chars is given and not None, remove characters in chars instead.')\n",
-      "{'attributes': {'trulens_eval@description': 'Return a copy of the string with '\n",
-      "                                            'trailing whitespace removed.\\n'\n",
-      "                                            '\\n'\n",
-      "                                            'If chars is given and not None, '\n",
-      "                                            'remove characters in chars '\n",
-      "                                            'instead.',\n",
-      "                'trulens_eval@error': None,\n",
-      "                'trulens_eval@inputs': {'data': 'Relevant chunk: HELLO'},\n",
-      "                'trulens_eval@output': 'RELEVANT CHUNK: HELLO',\n",
-      "                'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      "                'trulens_eval@span_type': 'SpanTool'},\n",
-      " 'attributes_metadata': {},\n",
-      " 'context': (4444476882659008121, 2775884631712611030, False, 0, [], True),\n",
-      " 'description': 'Return a copy of the string with trailing whitespace '\n",
-      "                'removed.\\n'\n",
-      "                '\\n'\n",
-      "                'If chars is given and not None, remove characters in chars '\n",
-      "                'instead.',\n",
-      " 'end_timestamp': 1714417146038280000,\n",
-      " 'error': None,\n",
-      " 'events': [],\n",
-      " 'inputs': {'data': 'Relevant chunk: HELLO'},\n",
-      " 'kind': <SpanKind.INTERNAL: 0>,\n",
-      " 'links': [((4444476882659008121, 11383545646059065267, False, 0, [], True),\n",
-      "            {'trulens_eval@relationship': 'parent'})],\n",
-      " 'name': 'invoke',\n",
-      " 'output': 'RELEVANT CHUNK: HELLO',\n",
-      " 'record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      " 'span_type': 'SpanTool',\n",
-      " 'start_timestamp': 1714417146023272000,\n",
-      " 'status': <StatusCode.OK: 1>,\n",
-      " 'status_description': None,\n",
-      " 'tags': []}\n",
-      "\n",
-      "SpanTool(name='invoke', kind=<SpanKind.INTERNAL: 0>, status=<StatusCode.OK: 1>, status_description=None, start_timestamp=1714417146057370000, end_timestamp=1714417146064115000, context=HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0xb98f15658cd7ae28, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x9dfa7a73d5cb8fb3, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanTool', 'trulens_eval@description': 'Return a copy of the string with trailing whitespace removed.\\n\\nIf chars is given and not None, remove characters in chars instead.', 'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c', 'trulens_eval@inputs': {'data': 'Relevant chunk: olleholleholleh'}, 'trulens_eval@output': 'relevant chunk: olleholleholleh', 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), ts=datetime.datetime(2024, 4, 29, 18, 59, 6, 452457), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11474992912, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 773577), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 802310)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.reranker, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_reranker.CustomReranker, id=11475027856, init_bindings=None), name='rerank'))], args={'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, rets=[['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 929746), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 975183)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[1], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: HELLO'}, rets='RELEVANT CHUNK: HELLO', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 23272), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 38280)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[0], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: olleholleholleh'}, rets='relevant chunk: olleholleholleh', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 57370), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64115)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context'))], args={'input': 'hello'}, rets=['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 694946), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64200)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 128985), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 154244)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11463407760, init_bindings=None), name='generate'))], args={'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, rets=\"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 223581), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 249975)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11474986832, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 311945), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 335659)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 426073), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452334)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), pid=41174, tid=15934086)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[0], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: olleholleholleh'}, rets='relevant chunk: olleholleholleh', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 57370), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64115)), pid=41174, tid=15934405), tags=[], span_type='SpanTool', record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', inputs={'data': 'Relevant chunk: olleholleholleh'}, output='relevant chunk: olleholleholleh', error=None, description='Return a copy of the string with trailing whitespace removed.\\n\\nIf chars is given and not None, remove characters in chars instead.')\n",
-      "{'attributes': {'trulens_eval@description': 'Return a copy of the string with '\n",
-      "                                            'trailing whitespace removed.\\n'\n",
-      "                                            '\\n'\n",
-      "                                            'If chars is given and not None, '\n",
-      "                                            'remove characters in chars '\n",
-      "                                            'instead.',\n",
-      "                'trulens_eval@error': None,\n",
-      "                'trulens_eval@inputs': {'data': 'Relevant chunk: '\n",
-      "                                                'olleholleholleh'},\n",
-      "                'trulens_eval@output': 'relevant chunk: olleholleholleh',\n",
-      "                'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      "                'trulens_eval@span_type': 'SpanTool'},\n",
-      " 'attributes_metadata': {},\n",
-      " 'context': (4444476882659008121, 13370929344585117224, False, 0, [], True),\n",
-      " 'description': 'Return a copy of the string with trailing whitespace '\n",
-      "                'removed.\\n'\n",
-      "                '\\n'\n",
-      "                'If chars is given and not None, remove characters in chars '\n",
-      "                'instead.',\n",
-      " 'end_timestamp': 1714417146064115000,\n",
-      " 'error': None,\n",
-      " 'events': [],\n",
-      " 'inputs': {'data': 'Relevant chunk: olleholleholleh'},\n",
-      " 'kind': <SpanKind.INTERNAL: 0>,\n",
-      " 'links': [((4444476882659008121, 11383545646059065267, False, 0, [], True),\n",
-      "            {'trulens_eval@relationship': 'parent'})],\n",
-      " 'name': 'invoke',\n",
-      " 'output': 'relevant chunk: olleholleholleh',\n",
-      " 'record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      " 'span_type': 'SpanTool',\n",
-      " 'start_timestamp': 1714417146057370000,\n",
-      " 'status': <StatusCode.OK: 1>,\n",
-      " 'status_description': None,\n",
-      " 'tags': []}\n",
-      "\n",
-      "SpanOther(name='get_context', kind=<SpanKind.INTERNAL: 0>, status=<StatusCode.OK: 1>, status_description=None, start_timestamp=1714417145694946000, end_timestamp=1714417146064200000, context=HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x9dfa7a73d5cb8fb3, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x0ff1d45c79519d0e, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanOther', 'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c', 'trulens_eval@inputs': {'input': 'hello'}, 'trulens_eval@output': ['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), ts=datetime.datetime(2024, 4, 29, 18, 59, 6, 452457), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11474992912, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 773577), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 802310)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.reranker, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_reranker.CustomReranker, id=11475027856, init_bindings=None), name='rerank'))], args={'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, rets=[['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 929746), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 975183)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[1], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: HELLO'}, rets='RELEVANT CHUNK: HELLO', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 23272), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 38280)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[0], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: olleholleholleh'}, rets='relevant chunk: olleholleholleh', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 57370), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64115)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context'))], args={'input': 'hello'}, rets=['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 694946), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64200)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 128985), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 154244)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11463407760, init_bindings=None), name='generate'))], args={'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, rets=\"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 223581), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 249975)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11474986832, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 311945), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 335659)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 426073), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452334)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), pid=41174, tid=15934086)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context'))], args={'input': 'hello'}, rets=['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 694946), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64200)), pid=41174, tid=15934086), tags=[], span_type='SpanOther', record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', inputs={'input': 'hello'}, output=['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], error=None)\n",
-      "{'attributes': {'trulens_eval@error': None,\n",
-      "                'trulens_eval@inputs': {'input': 'hello'},\n",
-      "                'trulens_eval@output': ['RELEVANT CHUNK: HELLO',\n",
-      "                                        'relevant chunk: olleholleholleh'],\n",
-      "                'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      "                'trulens_eval@span_type': 'SpanOther'},\n",
-      " 'attributes_metadata': {},\n",
-      " 'context': (4444476882659008121, 11383545646059065267, False, 0, [], True),\n",
-      " 'end_timestamp': 1714417146064200000,\n",
-      " 'error': None,\n",
-      " 'events': [],\n",
-      " 'inputs': {'input': 'hello'},\n",
-      " 'kind': <SpanKind.INTERNAL: 0>,\n",
-      " 'links': [((4444476882659008121, 1148932873593658638, False, 0, [], True),\n",
-      "            {'trulens_eval@relationship': 'parent'})],\n",
-      " 'name': 'get_context',\n",
-      " 'output': ['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'],\n",
-      " 'record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      " 'span_type': 'SpanOther',\n",
-      " 'start_timestamp': 1714417145694946000,\n",
-      " 'status': <StatusCode.OK: 1>,\n",
-      " 'status_description': None,\n",
-      " 'tags': []}\n",
-      "\n",
-      "SpanMemory(name='remember', kind=<SpanKind.INTERNAL: 0>, status=<StatusCode.OK: 1>, status_description=None, start_timestamp=1714417146128985000, end_timestamp=1714417146154244000, context=HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x14ae492dd25203b8, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x0ff1d45c79519d0e, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanMemory', 'trulens_eval@memory_type': 'dummy-memory', 'trulens_eval@remembered': 'hello', 'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c', 'trulens_eval@inputs': {'data': 'hello'}, 'trulens_eval@output': None, 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), ts=datetime.datetime(2024, 4, 29, 18, 59, 6, 452457), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11474992912, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 773577), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 802310)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.reranker, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_reranker.CustomReranker, id=11475027856, init_bindings=None), name='rerank'))], args={'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, rets=[['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 929746), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 975183)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[1], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: HELLO'}, rets='RELEVANT CHUNK: HELLO', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 23272), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 38280)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[0], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: olleholleholleh'}, rets='relevant chunk: olleholleholleh', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 57370), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64115)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context'))], args={'input': 'hello'}, rets=['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 694946), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64200)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 128985), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 154244)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11463407760, init_bindings=None), name='generate'))], args={'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, rets=\"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 223581), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 249975)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11474986832, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 311945), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 335659)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 426073), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452334)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), pid=41174, tid=15934086)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 128985), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 154244)), pid=41174, tid=15934086), tags=[], span_type='SpanMemory', record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', inputs={'data': 'hello'}, output=None, error=None, memory_type='dummy-memory', remembered='hello')\n",
-      "{'attributes': {'trulens_eval@error': None,\n",
-      "                'trulens_eval@inputs': {'data': 'hello'},\n",
-      "                'trulens_eval@memory_type': 'dummy-memory',\n",
-      "                'trulens_eval@output': None,\n",
-      "                'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      "                'trulens_eval@remembered': 'hello',\n",
-      "                'trulens_eval@span_type': 'SpanMemory'},\n",
-      " 'attributes_metadata': {},\n",
-      " 'context': (4444476882659008121, 1490208987857159096, False, 0, [], True),\n",
-      " 'end_timestamp': 1714417146154244000,\n",
-      " 'error': None,\n",
-      " 'events': [],\n",
-      " 'inputs': {'data': 'hello'},\n",
-      " 'kind': <SpanKind.INTERNAL: 0>,\n",
-      " 'links': [((4444476882659008121, 1148932873593658638, False, 0, [], True),\n",
-      "            {'trulens_eval@relationship': 'parent'})],\n",
-      " 'memory_type': 'dummy-memory',\n",
-      " 'name': 'remember',\n",
-      " 'output': None,\n",
-      " 'record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      " 'remembered': 'hello',\n",
-      " 'span_type': 'SpanMemory',\n",
-      " 'start_timestamp': 1714417146128985000,\n",
-      " 'status': <StatusCode.OK: 1>,\n",
-      " 'status_description': None,\n",
-      " 'tags': []}\n",
-      "\n",
-      "SpanLLM(name='generate', kind=<SpanKind.INTERNAL: 0>, status=<StatusCode.OK: 1>, status_description=None, start_timestamp=1714417146223581000, end_timestamp=1714417146249975000, context=HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x45d94a1cdc26b24e, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x0ff1d45c79519d0e, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanLLM', 'trulens_eval@model_name': 'derp', 'trulens_eval@model_type': 'HerpDerp', 'trulens_eval@temperature': 0.5, 'trulens_eval@input_messages': [{'content': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}], 'trulens_eval@input_token_count': 5, 'trulens_eval@output_messages': [{'content': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}], 'trulens_eval@output_token_count': 13, 'trulens_eval@cost': 42.0, 'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c', 'trulens_eval@inputs': {'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, 'trulens_eval@output': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), ts=datetime.datetime(2024, 4, 29, 18, 59, 6, 452457), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11474992912, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 773577), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 802310)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.reranker, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_reranker.CustomReranker, id=11475027856, init_bindings=None), name='rerank'))], args={'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, rets=[['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 929746), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 975183)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[1], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: HELLO'}, rets='RELEVANT CHUNK: HELLO', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 23272), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 38280)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[0], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: olleholleholleh'}, rets='relevant chunk: olleholleholleh', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 57370), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64115)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context'))], args={'input': 'hello'}, rets=['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 694946), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64200)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 128985), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 154244)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11463407760, init_bindings=None), name='generate'))], args={'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, rets=\"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 223581), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 249975)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11474986832, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 311945), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 335659)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 426073), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452334)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), pid=41174, tid=15934086)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11463407760, init_bindings=None), name='generate'))], args={'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, rets=\"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 223581), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 249975)), pid=41174, tid=15934086), tags=[], span_type='SpanLLM', record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', inputs={'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, output=\"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", error=None, model_name='derp', model_type='HerpDerp', temperature=0.5, input_messages=[{'content': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}], input_token_count=5, output_messages=[{'content': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}], output_token_count=13, cost=42.0)\n",
-      "{'attributes': {'trulens_eval@cost': 42.0,\n",
-      "                'trulens_eval@error': None,\n",
-      "                'trulens_eval@input_messages': [{'content': 'RELEVANT CHUNK: '\n",
-      "                                                            'HELLO,relevant '\n",
-      "                                                            'chunk: '\n",
-      "                                                            'olleholleholleh'}],\n",
-      "                'trulens_eval@input_token_count': 5,\n",
-      "                'trulens_eval@inputs': {'prompt': 'RELEVANT CHUNK: '\n",
-      "                                                  'HELLO,relevant chunk: '\n",
-      "                                                  'olleholleholleh'},\n",
-      "                'trulens_eval@model_name': 'derp',\n",
-      "                'trulens_eval@model_type': 'HerpDerp',\n",
-      "                'trulens_eval@output': \"Generating from 'RELEVANT CHUNK: \"\n",
-      "                                       \"HELLO,relevant chunk: olleholleholleh' \"\n",
-      "                                       'and 56 bytes with temperature 0.5',\n",
-      "                'trulens_eval@output_messages': [{'content': 'Generating from '\n",
-      "                                                             \"'RELEVANT CHUNK: \"\n",
-      "                                                             'HELLO,relevant '\n",
-      "                                                             'chunk: '\n",
-      "                                                             \"olleholleholleh' \"\n",
-      "                                                             'and 56 bytes '\n",
-      "                                                             'with temperature '\n",
-      "                                                             '0.5'}],\n",
-      "                'trulens_eval@output_token_count': 13,\n",
-      "                'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      "                'trulens_eval@span_type': 'SpanLLM',\n",
-      "                'trulens_eval@temperature': 0.5},\n",
-      " 'attributes_metadata': {},\n",
-      " 'context': (4444476882659008121, 5033135546376303182, False, 0, [], True),\n",
-      " 'cost': 42.0,\n",
-      " 'end_timestamp': 1714417146249975000,\n",
-      " 'error': None,\n",
-      " 'events': [],\n",
-      " 'input_messages': [{'content': 'RELEVANT CHUNK: HELLO,relevant chunk: '\n",
-      "                                'olleholleholleh'}],\n",
-      " 'input_token_count': 5,\n",
-      " 'inputs': {'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'},\n",
-      " 'kind': <SpanKind.INTERNAL: 0>,\n",
-      " 'links': [((4444476882659008121, 1148932873593658638, False, 0, [], True),\n",
-      "            {'trulens_eval@relationship': 'parent'})],\n",
-      " 'model_name': 'derp',\n",
-      " 'model_type': 'HerpDerp',\n",
-      " 'name': 'generate',\n",
-      " 'output': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: \"\n",
-      "           \"olleholleholleh' and 56 bytes with temperature 0.5\",\n",
-      " 'output_messages': [{'content': \"Generating from 'RELEVANT CHUNK: \"\n",
-      "                                 \"HELLO,relevant chunk: olleholleholleh' and \"\n",
-      "                                 '56 bytes with temperature 0.5'}],\n",
-      " 'output_token_count': 13,\n",
-      " 'record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      " 'span_type': 'SpanLLM',\n",
-      " 'start_timestamp': 1714417146223581000,\n",
-      " 'status': <StatusCode.OK: 1>,\n",
-      " 'status_description': None,\n",
-      " 'tags': [],\n",
-      " 'temperature': 0.5}\n",
-      "\n",
-      "SpanOther(name='fill', kind=<SpanKind.INTERNAL: 0>, status=<StatusCode.OK: 1>, status_description=None, start_timestamp=1714417146311945000, end_timestamp=1714417146335659000, context=HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x44740a38f701b686, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x0ff1d45c79519d0e, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanOther', 'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c', 'trulens_eval@inputs': {'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, 'trulens_eval@output': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), ts=datetime.datetime(2024, 4, 29, 18, 59, 6, 452457), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11474992912, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 773577), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 802310)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.reranker, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_reranker.CustomReranker, id=11475027856, init_bindings=None), name='rerank'))], args={'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, rets=[['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 929746), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 975183)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[1], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: HELLO'}, rets='RELEVANT CHUNK: HELLO', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 23272), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 38280)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[0], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: olleholleholleh'}, rets='relevant chunk: olleholleholleh', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 57370), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64115)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context'))], args={'input': 'hello'}, rets=['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 694946), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64200)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 128985), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 154244)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11463407760, init_bindings=None), name='generate'))], args={'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, rets=\"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 223581), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 249975)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11474986832, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 311945), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 335659)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 426073), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452334)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), pid=41174, tid=15934086)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11474986832, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 311945), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 335659)), pid=41174, tid=15934086), tags=[], span_type='SpanOther', record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', inputs={'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, output=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None)\n",
-      "{'attributes': {'trulens_eval@error': None,\n",
-      "                'trulens_eval@inputs': {'answer': \"Generating from 'RELEVANT \"\n",
-      "                                                  'CHUNK: HELLO,relevant '\n",
-      "                                                  \"chunk: olleholleholleh' and \"\n",
-      "                                                  '56 bytes with temperature '\n",
-      "                                                  '0.5',\n",
-      "                                        'question': 'hello'},\n",
-      "                'trulens_eval@output': 'The answer to hello is probably '\n",
-      "                                       \"Generating from 'RELEVANT CHUNK: \"\n",
-      "                                       \"HELLO,relevant chunk: olleholleholleh' \"\n",
-      "                                       'and 56 bytes with temperature 0.5 or '\n",
-      "                                       'something ...',\n",
-      "                'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      "                'trulens_eval@span_type': 'SpanOther'},\n",
-      " 'attributes_metadata': {},\n",
-      " 'context': (4444476882659008121, 4932578731656066694, False, 0, [], True),\n",
-      " 'end_timestamp': 1714417146335659000,\n",
-      " 'error': None,\n",
-      " 'events': [],\n",
-      " 'inputs': {'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: \"\n",
-      "                      \"olleholleholleh' and 56 bytes with temperature 0.5\",\n",
-      "            'question': 'hello'},\n",
-      " 'kind': <SpanKind.INTERNAL: 0>,\n",
-      " 'links': [((4444476882659008121, 1148932873593658638, False, 0, [], True),\n",
-      "            {'trulens_eval@relationship': 'parent'})],\n",
-      " 'name': 'fill',\n",
-      " 'output': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: \"\n",
-      "           \"HELLO,relevant chunk: olleholleholleh' and 56 bytes with \"\n",
-      "           'temperature 0.5 or something ...',\n",
-      " 'record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      " 'span_type': 'SpanOther',\n",
-      " 'start_timestamp': 1714417146311945000,\n",
-      " 'status': <StatusCode.OK: 1>,\n",
-      " 'status_description': None,\n",
-      " 'tags': []}\n",
-      "\n",
-      "SpanMemory(name='remember', kind=<SpanKind.INTERNAL: 0>, status=<StatusCode.OK: 1>, status_description=None, start_timestamp=1714417146426073000, end_timestamp=1714417146452334000, context=HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x3860b1e53abfc055, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x0ff1d45c79519d0e, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanMemory', 'trulens_eval@memory_type': 'dummy-memory', 'trulens_eval@remembered': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", 'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c', 'trulens_eval@inputs': {'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, 'trulens_eval@output': None, 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), ts=datetime.datetime(2024, 4, 29, 18, 59, 6, 452457), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11474992912, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 773577), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 802310)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.reranker, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_reranker.CustomReranker, id=11475027856, init_bindings=None), name='rerank'))], args={'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, rets=[['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 929746), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 975183)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[1], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: HELLO'}, rets='RELEVANT CHUNK: HELLO', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 23272), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 38280)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[0], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: olleholleholleh'}, rets='relevant chunk: olleholleholleh', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 57370), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64115)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context'))], args={'input': 'hello'}, rets=['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 694946), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64200)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 128985), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 154244)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11463407760, init_bindings=None), name='generate'))], args={'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, rets=\"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 223581), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 249975)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11474986832, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 311945), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 335659)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 426073), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452334)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), pid=41174, tid=15934086)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 426073), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452334)), pid=41174, tid=15934086), tags=[], span_type='SpanMemory', record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', inputs={'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, output=None, error=None, memory_type='dummy-memory', remembered=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\")\n",
-      "{'attributes': {'trulens_eval@error': None,\n",
-      "                'trulens_eval@inputs': {'data': 'The answer to hello is '\n",
-      "                                                'probably Generating from '\n",
-      "                                                \"'RELEVANT CHUNK: \"\n",
-      "                                                'HELLO,relevant chunk: '\n",
-      "                                                \"olleholleholleh' and 56 bytes \"\n",
-      "                                                'with temperature 0.5 or '\n",
-      "                                                'something ...'},\n",
-      "                'trulens_eval@memory_type': 'dummy-memory',\n",
-      "                'trulens_eval@output': None,\n",
-      "                'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      "                'trulens_eval@remembered': 'The answer to hello is probably '\n",
-      "                                           \"Generating from 'RELEVANT CHUNK: \"\n",
-      "                                           'HELLO,relevant chunk: '\n",
-      "                                           \"olleholleholleh' and 56 bytes with \"\n",
-      "                                           'temperature 0.5 or something ...',\n",
-      "                'trulens_eval@span_type': 'SpanMemory'},\n",
-      " 'attributes_metadata': {},\n",
-      " 'context': (4444476882659008121, 4062442461979459669, False, 0, [], True),\n",
-      " 'end_timestamp': 1714417146452334000,\n",
-      " 'error': None,\n",
-      " 'events': [],\n",
-      " 'inputs': {'data': \"The answer to hello is probably Generating from 'RELEVANT \"\n",
-      "                    \"CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 \"\n",
-      "                    'bytes with temperature 0.5 or something ...'},\n",
-      " 'kind': <SpanKind.INTERNAL: 0>,\n",
-      " 'links': [((4444476882659008121, 1148932873593658638, False, 0, [], True),\n",
-      "            {'trulens_eval@relationship': 'parent'})],\n",
-      " 'memory_type': 'dummy-memory',\n",
-      " 'name': 'remember',\n",
-      " 'output': None,\n",
-      " 'record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      " 'remembered': \"The answer to hello is probably Generating from 'RELEVANT \"\n",
-      "               \"CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes \"\n",
-      "               'with temperature 0.5 or something ...',\n",
-      " 'span_type': 'SpanMemory',\n",
-      " 'start_timestamp': 1714417146426073000,\n",
-      " 'status': <StatusCode.OK: 1>,\n",
-      " 'status_description': None,\n",
-      " 'tags': []}\n",
-      "\n",
-      "SpanOther(name='respond_to_query', kind=<SpanKind.INTERNAL: 0>, status=<StatusCode.OK: 1>, status_description=None, start_timestamp=1714417145465477000, end_timestamp=1714417146452397000, context=HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0x0ff1d45c79519d0e, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0x00000000000000003dadf42d4b535e79, span_id=0xc461bfb604e9f8cc, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanOther', 'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c', 'trulens_eval@inputs': {'input': 'hello'}, 'trulens_eval@output': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), ts=datetime.datetime(2024, 4, 29, 18, 59, 6, 452457), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11474992912, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 773577), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 802310)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.reranker, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_reranker.CustomReranker, id=11475027856, init_bindings=None), name='rerank'))], args={'query_text': 'hello', 'chunks': ['Relevant chunk: HELLO', 'Relevant chunk: olleholleholleh'], 'chunk_scores': None}, rets=[['Relevant chunk: HELLO', 16.0], ['Relevant chunk: olleholleholleh', 26.0]], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 929746), end_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 975183)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[1], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: HELLO'}, rets='RELEVANT CHUNK: HELLO', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 23272), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 38280)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context')), RecordAppCallMethod(path=Lens().app.tools[0], method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_tool.CustomTool, id=11475026896, init_bindings=None), name='invoke'))], args={'data': 'Relevant chunk: olleholleholleh'}, rets='relevant chunk: olleholleholleh', error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 57370), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64115)), pid=41174, tid=15934405), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='get_context'))], args={'input': 'hello'}, rets=['RELEVANT CHUNK: HELLO', 'relevant chunk: olleholleholleh'], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 694946), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 64200)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 128985), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 154244)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11463407760, init_bindings=None), name='generate'))], args={'prompt': 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh'}, rets=\"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 223581), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 249975)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11474986832, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5\"}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 311945), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 335659)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11474988304, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 426073), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452334)), pid=41174, tid=15934086), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), pid=41174, tid=15934086)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11469952144, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 29, 18, 59, 5, 465477), end_time=datetime.datetime(2024, 4, 29, 18, 59, 6, 452397)), pid=41174, tid=15934086), tags=[], span_type='SpanOther', record_id='record_hash_c7c1852d78e621164a9d2965e2e5686c', inputs={'input': 'hello'}, output=\"The answer to hello is probably Generating from 'RELEVANT CHUNK: HELLO,relevant chunk: olleholleholleh' and 56 bytes with temperature 0.5 or something ...\", error=None)\n",
-      "{'attributes': {'trulens_eval@error': None,\n",
-      "                'trulens_eval@inputs': {'input': 'hello'},\n",
-      "                'trulens_eval@output': 'The answer to hello is probably '\n",
-      "                                       \"Generating from 'RELEVANT CHUNK: \"\n",
-      "                                       \"HELLO,relevant chunk: olleholleholleh' \"\n",
-      "                                       'and 56 bytes with temperature 0.5 or '\n",
-      "                                       'something ...',\n",
-      "                'trulens_eval@record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      "                'trulens_eval@span_type': 'SpanOther'},\n",
-      " 'attributes_metadata': {},\n",
-      " 'context': (4444476882659008121, 1148932873593658638, False, 0, [], True),\n",
-      " 'end_timestamp': 1714417146452397000,\n",
-      " 'error': None,\n",
-      " 'events': [],\n",
-      " 'inputs': {'input': 'hello'},\n",
-      " 'kind': <SpanKind.INTERNAL: 0>,\n",
-      " 'links': [((4444476882659008121, 14150802292662204620, False, 0, [], True),\n",
-      "            {'trulens_eval@relationship': 'parent'})],\n",
-      " 'name': 'respond_to_query',\n",
-      " 'output': \"The answer to hello is probably Generating from 'RELEVANT CHUNK: \"\n",
-      "           \"HELLO,relevant chunk: olleholleholleh' and 56 bytes with \"\n",
-      "           'temperature 0.5 or something ...',\n",
-      " 'record_id': 'record_hash_c7c1852d78e621164a9d2965e2e5686c',\n",
-      " 'span_type': 'SpanOther',\n",
-      " 'start_timestamp': 1714417145465477000,\n",
-      " 'status': <StatusCode.OK: 1>,\n",
-      " 'status_description': None,\n",
-      " 'tags': []}\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/opt/homebrew/Caskroom/miniconda/base/envs/py311_trulens/lib/python3.11/site-packages/pydantic/main.py:314: UserWarning: Pydantic serializer warnings:\n",
-      "  Expected `Union[str, bool, int, float, json-or-python[json=list[str], python=list[str]], json-or-python[json=list[bool], python=list[bool]], json-or-python[json=list[int], python=list[int]], json-or-python[json=list[float], python=list[float]]]` but got `list` - serialized value may not be as expected\n",
-      "  Expected `Union[str, bool, int, float, json-or-python[json=list[str], python=list[str]], json-or-python[json=list[bool], python=list[bool]], json-or-python[json=list[int], python=list[int]], json-or-python[json=list[float], python=list[float]]]` but got `dict` - serialized value may not be as expected\n",
-      "  return self.__pydantic_serializer__.to_python(\n",
-      "/opt/homebrew/Caskroom/miniconda/base/envs/py311_trulens/lib/python3.11/site-packages/pydantic/main.py:314: UserWarning: Pydantic serializer warnings:\n",
-      "  Expected `Union[str, bool, int, float, json-or-python[json=list[str], python=list[str]], json-or-python[json=list[bool], python=list[bool]], json-or-python[json=list[int], python=list[int]], json-or-python[json=list[float], python=list[float]]]` but got `dict` - serialized value may not be as expected\n",
-      "  Expected `Union[str, bool, int, float, json-or-python[json=list[str], python=list[str]], json-or-python[json=list[bool], python=list[bool]], json-or-python[json=list[int], python=list[int]], json-or-python[json=list[float], python=list[float]]]` but got `list` - serialized value may not be as expected\n",
-      "  return self.__pydantic_serializer__.to_python(\n",
-      "/opt/homebrew/Caskroom/miniconda/base/envs/py311_trulens/lib/python3.11/site-packages/pydantic/main.py:314: UserWarning: Pydantic serializer warnings:\n",
-      "  Expected `Union[str, bool, int, float, json-or-python[json=list[str], python=list[str]], json-or-python[json=list[bool], python=list[bool]], json-or-python[json=list[int], python=list[int]], json-or-python[json=list[float], python=list[float]]]` but got `dict` - serialized value may not be as expected\n",
-      "  return self.__pydantic_serializer__.to_python(\n",
-      "/opt/homebrew/Caskroom/miniconda/base/envs/py311_trulens/lib/python3.11/site-packages/pydantic/main.py:314: UserWarning: Pydantic serializer warnings:\n",
-      "  Expected `Union[str, bool, int, float, json-or-python[json=list[str], python=list[str]], json-or-python[json=list[bool], python=list[bool]], json-or-python[json=list[int], python=list[int]], json-or-python[json=list[float], python=list[float]]]` but got `list` - serialized value may not be as expected\n",
-      "  Expected `Union[str, bool, int, float, json-or-python[json=list[str], python=list[str]], json-or-python[json=list[bool], python=list[bool]], json-or-python[json=list[int], python=list[int]], json-or-python[json=list[float], python=list[float]]]` but got `list` - serialized value may not be as expected\n",
-      "  Expected `Union[str, bool, int, float, json-or-python[json=list[str], python=list[str]], json-or-python[json=list[bool], python=list[bool]], json-or-python[json=list[int], python=list[int]], json-or-python[json=list[float], python=list[float]]]` but got `dict` - serialized value may not be as expected\n",
-      "  return self.__pydantic_serializer__.to_python(\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "for span in spans:\n",
     "    pprint(span)\n",
diff --git a/trulens_eval/trulens_eval/trace/span.py b/trulens_eval/trulens_eval/trace/span.py
index f693d0211..1759e33cd 100644
--- a/trulens_eval/trulens_eval/trace/span.py
+++ b/trulens_eval/trulens_eval/trace/span.py
@@ -21,6 +21,7 @@
 from trulens_eval import trace as mod_trace
 from trulens_eval.schema import record as mod_record_schema
 from trulens_eval.utils import containers as mod_container_utils
+from trulens_eval.utils.serial import JSON
 
 logger = getLogger(__name__)
 

From 66905dc6ecf77a43467b6317cb016c32c4b40bdc Mon Sep 17 00:00:00 2001
From: Piotr Mardziel <piotrm@gmail.com>
Date: Wed, 1 May 2024 16:46:48 -0700
Subject: [PATCH 2/4] spans work

---
 docs/trulens_eval/api/trace/index.md          |   4 +-
 .../examples/experimental/spans_example.ipynb | 369 ++++++++++++++----
 trulens_eval/trulens_eval/trace/__init__.py   |   6 +-
 trulens_eval/trulens_eval/trace/span.py       |  28 +-
 trulens_eval/trulens_eval/utils/serial.py     |   8 +-
 5 files changed, 318 insertions(+), 97 deletions(-)

diff --git a/docs/trulens_eval/api/trace/index.md b/docs/trulens_eval/api/trace/index.md
index d08dc68ce..c814d3007 100644
--- a/docs/trulens_eval/api/trace/index.md
+++ b/docs/trulens_eval/api/trace/index.md
@@ -1,3 +1,5 @@
 # Trace
 
-::: trulens_eval.trace
\ No newline at end of file
+::: trulens_eval.trace
+
+::: trulens_eval.trace.category
diff --git a/trulens_eval/examples/experimental/spans_example.ipynb b/trulens_eval/examples/experimental/spans_example.ipynb
index 5450f9cb0..7f9b2eea5 100644
--- a/trulens_eval/examples/experimental/spans_example.ipynb
+++ b/trulens_eval/examples/experimental/spans_example.ipynb
@@ -54,11 +54,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 141,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from typing import Union, Dict, List, Mapping, Sequence, TypeVar, Generic, Optional\n",
+    "from typing import Union, Dict, List, Mapping, Sequence, TypeVar, Generic, Optional, Type, Iterator\n",
     "\n",
     "TPyJSONBase = Union[str, int, float, bool]\n",
     "\n",
@@ -78,145 +78,354 @@
     "]\n",
     "\"\"\"\n",
     "\n",
-    "from trulens_eval.utils.serial import Lens\n",
+    "from trulens_eval.utils.serial import Lens, GetIndex, GetItem, Step\n",
     "\n",
     "T = TypeVar(\"T\")\n",
+    "C = TypeVar(\"C\")\n",
+    "\n",
+    "ValueOrContainer = Union[T, 'MappedContainer[T, C]']\n",
+    "\n",
+    "class MappedContainer(Generic[T, C]): # will behave like C[T]\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        value: Optional[C] = None,\n",
+    "        container_class: Optional[type] = None,\n",
+    "        global_store: Optional[Dict[Lens, ot_types.AttributeValue]] = None,\n",
+    "        lens: Lens = None,\n",
+    "    ):\n",
+    "\n",
+    "        if global_store is None:\n",
+    "            global_store: Dict[Lens, ot_types.AttributeValue] = {}\n",
+    "\n",
+    "        self.global_store = global_store\n",
     "\n",
-    "class LensMappedDict(Generic[T], Dict[str, T]):\n",
-    "    def __init__(self, store: Optional[dict] = None, lens: Lens = None):\n",
-    "        if store is None:\n",
-    "            store = {}\n",
-    "        self.store = store\n",
+    "        self.step_store: Dict[Step, ValueOrContainer[T]] = {}\n",
     "\n",
     "        if lens is None:\n",
     "            lens = Lens() \n",
     "        self.lens = lens\n",
     "\n",
-    "        self.relative_store = {}\n",
+    "        if value is None:\n",
+    "            if container_class is None:\n",
+    "                raise ValueError(\"Cannot have a None value and None container class.\")\n",
+    "            value = container_class()\n",
     "\n",
-    "    def __repr__(self):\n",
-    "        return repr(self.relative_store)\n",
+    "        if container_class is None:\n",
+    "            container_class = type(value)\n",
+    "        \n",
+    "        self.container_class = container_class\n",
     "\n",
-    "    def __str__(self):\n",
-    "        return str(self.relative_store)\n",
+    "        self.map(value)\n",
     "\n",
-    "    def unmap_value(self, value: Union[T, 'LensMappedDict[T]']) -> T:\n",
-    "        if isinstance(value, LensMappedDict):\n",
-    "            ret = {}\n",
-    "            for k, v in value.relative_store.items():\n",
-    "                ret[k] = value.unmap_value(v)\n",
+    "    @staticmethod\n",
+    "    def make_mapped_container(\n",
+    "        value: Optional[C] = None,\n",
+    "        container_class: Optional[type] = None,\n",
+    "        global_store: Optional[Dict[Lens, ot_types.AttributeValue]] = None,\n",
+    "        lens: Lens = None\n",
+    "    ) -> ValueOrContainer[T]:\n",
+    "        mtype = MappedContainer.mappedtype_for_value(value)\n",
     "\n",
-    "            return ret\n",
-    "            \n",
-    "        elif isinstance(value, TPyJSONBase):\n",
+    "        if mtype is None:\n",
     "            return value\n",
+    "        \n",
+    "        return mtype(\n",
+    "            global_store=global_store,\n",
+    "            value=value,\n",
+    "            lens=lens\n",
+    "        )\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def mappedtype_for_value(value: T) -> Optional[Type[MappedContainer[T, C]]]:\n",
+    "        if isinstance(value, Mapping):\n",
+    "            return MappedDict\n",
+    "        elif isinstance(value, Sequence):\n",
+    "            return MappedList\n",
+    "        else:\n",
+    "            return None\n",
     "\n",
-    "        elif value is None:\n",
-    "            return value\n",
+    "    def __iter__(self):\n",
+    "        raise TypeError(\"Cannot iterate over a non-container.\")\n",
     "\n",
-    "        else:\n",
-    "            raise ValueError(f\"Unexpected type: {type(value)}\")\n",
+    "    def __str__(self):\n",
+    "        return str(self.unmap())\n",
     "\n",
-    "    def map_value(self, lens: Lens, value: T) -> Union[T, 'LensMappedDict[T]']:\n",
-    "        if isinstance(value, TPyJSONBase):\n",
-    "            return value\n",
+    "    def __repr__(self):\n",
+    "        return repr(self.unmap())\n",
     "\n",
-    "        elif isinstance(value, Dict):\n",
-    "            temp = LensMappedDict(store=self.store, lens=lens)\n",
-    "            for k, v in value.items():\n",
-    "                temp[k] = v\n",
-    "            return temp\n",
+    "    def map(self, value: T, lens: Optional[Lens] = None) -> None:\n",
+    "        if lens is None:\n",
+    "            lens = self.lens\n",
+    "\n",
+    "        if isinstance(value, TPyJSONBase):\n",
+    "            self.global_store[lens] = value\n",
     "\n",
     "        elif isinstance(value, Sequence):\n",
-    "            if len(value) == 0:\n",
-    "                return value\n",
-    "            typ = type(value[0])\n",
+    "            for i in range(len(value)):\n",
+    "                step = GetIndex(index=i)\n",
+    "                subcontainer = MappedContainer.make_mapped_container(\n",
+    "                    global_store=self.global_store,\n",
+    "                    lens=self.lens[step],\n",
+    "                    value=value[i]\n",
+    "                )\n",
+    "                self.step_store[step] = subcontainer\n",
+    "\n",
+    "        elif isinstance(value, Mapping):\n",
+    "            for k, v in value.items():\n",
+    "                step = GetItem(item=k)\n",
+    "                subcontainer = MappedContainer.make_mapped_container(\n",
+    "                    global_store=self.global_store,\n",
+    "                    lens=self.lens[step],\n",
+    "                    value=v\n",
+    "                )\n",
+    "                self.step_store[step] = subcontainer\n",
     "\n",
-    "            if isinstance(value[0], TPyJSON) and all(isinstance(x, typ) for x in value):\n",
-    "                return value\n",
+    "        else:\n",
+    "            raise TypeError(f\"Unexpected type: {type(value)}\")\n",
     "\n",
-    "            temp = LensMappedDict(store=self.store, lens=lens)\n",
-    "            for i in range(len(value)):\n",
-    "                temp[i] = value[i]\n",
+    "        return self\n",
     "\n",
-    "            return temp\n",
+    "    def __getitem__(self, key: str) -> T:\n",
+    "        if self.container_class is None:\n",
+    "            raise TypeError(\"Cannot get item on a non-container.\")\n",
     "\n",
+    "        if self.container_class is dict:\n",
+    "            step = GetItem(item=key)\n",
+    "        elif self.container_class is list:\n",
+    "            step = GetIndex(index=key)\n",
     "        else:\n",
-    "            raise ValueError(f\"Unexpected type: {type(value)}\")\n",
+    "            raise TypeError(\"Unexpected container type.\")\n",
+    "\n",
+    "        return self.step_store.get(step)\n",
     "\n",
-    "    def __getitem__(self, key: str, default: Optional[T] = None) -> T:\n",
-    "        return self.relative_store.get(key, default)\n",
+    "    def __del__(self) -> None:\n",
+    "        if self.container_class is None:\n",
+    "            del self.global_store[self.lens]\n",
+    "            return\n",
+    "\n",
+    "        for step, val in self.step_store.items():\n",
+    "            del val\n",
     "\n",
     "    def __setitem__(self, key: str, value: T) -> None:\n",
-    "        lens = self.lens[key]\n",
-    "        mapped_val = self.map_value(lens, value)\n",
+    "        if self.container_class is None:\n",
+    "            raise TypeError(\"Cannot set item on a non-container.\")\n",
+    "\n",
+    "        if self.container_class is dict:\n",
+    "            step = GetItem(item=key)\n",
+    "        elif self.container_class is list:\n",
+    "            step = GetIndex(index=key)\n",
+    "        else:\n",
+    "            raise TypeError(\"Unexpected container type.\")\n",
     "\n",
-    "        if key in self.relative_store:\n",
-    "            # Need to delete as there might be multiple store lenses associated\n",
-    "            # with this key.\n",
+    "        if step in self.step_store:\n",
     "            del self[key]\n",
-    "        \n",
-    "        self.store[lens] = mapped_val\n",
-    "        self.relative_store[key] = mapped_val\n",
     "\n",
-    "    def __delitem__(self, key: str) -> None:\n",
-    "        lens = self.lens[key]\n",
-    "        val = self.relative_store[key]\n",
+    "        if isinstance(value, TPyJSONBase):\n",
+    "            mapped_value = value\n",
+    "            self.global_store[self.lens[step]] = value\n",
     "\n",
-    "        if isinstance(val, LensMappedDict):\n",
-    "            del val\n",
+    "        elif isinstance(value, (Mapping, Sequence)):\n",
+    "            mapped_value = MappedContainer.make_mapped_container(\n",
+    "                global_store=self.global_store,\n",
+    "                value=value,\n",
+    "                lens=self.lens[step]\n",
+    "            )\n",
     "\n",
-    "        del self.store[lens]\n",
-    "        del self.relative_store[key]\n",
+    "        else:\n",
+    "            raise TypeError(f\"Unexpected type: {type(value)}\")\n",
+    "\n",
+    "        self.step_store[step] = mapped_value\n",
+    "\n",
+    "    def map_value(self, value: T) -> ValueOrContainer[T]:\n",
+    "        return MappedContainer.make_mapped_container(\n",
+    "            global_store=self.global_store, value=value, lens=self.lens\n",
+    "        )\n",
     "\n",
-    "    def __del__(self):\n",
-    "        for k, v in list(self.relative_store.items()):\n",
-    "            sublens = self.lens[k]\n",
+    "    def unmap(self) -> T:\n",
+    "        return MappedContainer.unmap_value(self)\n",
     "\n",
-    "            if hasattr(v, \"__del__\"):\n",
-    "                del v\n",
+    "    @staticmethod\n",
+    "    def unmap_value(value: ValueOrContainer[T]) -> T:\n",
+    "        if isinstance(value, TPyJSONBase):\n",
+    "            return value\n",
     "\n",
-    "            del self.relative_store[k]\n",
-    "            del self.store[sublens]\n",
-    "\n"
+    "        if isinstance(value, MappedContainer):\n",
+    "            if value.container_class is None:\n",
+    "                return value.global_store[value.lens]\n",
+    "\n",
+    "            container = value.container_class()\n",
+    "            for step, v in value.step_store.items():\n",
+    "                container = step.set(container, MappedContainer.unmap_value(v))\n",
+    "            return container\n",
+    "        else:\n",
+    "            raise TypeError(f\"Unexpected type: {type(value)}\")\n",
+    "\n",
+    "class MappedList(MappedContainer[T, List], List[T]):\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        value: Optional[List[T]] = None,\n",
+    "        global_store: Optional[Dict[Lens, ot_types.AttributeValue]] = None,\n",
+    "        lens: Lens = None,\n",
+    "    ):\n",
+    "        super().__init__(\n",
+    "            value=value,\n",
+    "            global_store=global_store,\n",
+    "            lens=lens,\n",
+    "            container_class=list\n",
+    "        )\n",
+    "\n",
+    "    def __iter__(self) -> Iterator[T]:\n",
+    "        for step, value in self.step_store.items():\n",
+    "            yield value\n",
+    "\n",
+    "    def __delitem__(self, key: str) -> None:\n",
+    "        step = GetIndex(index=key)\n",
+    "    \n",
+    "        if not step in self.step_store:\n",
+    "            raise ValueError(f\"Index not in mapped list: {step}\")\n",
+    "\n",
+    "        val = self.step_store[step]\n",
+    "        del val\n",
+    "        del self.step_store[step]\n",
+    "\n",
+    "class MappedDict(MappedContainer[T, Dict], Dict[str, T]):\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        value: Optional[Dict[str, T]] = None,\n",
+    "        global_store: Optional[Dict[Lens, ot_types.AttributeValue]] = None,\n",
+    "        lens: Lens = None,\n",
+    "    ):\n",
+    "        super().__init__(\n",
+    "            value=value,\n",
+    "            global_store=global_store,\n",
+    "            lens=lens,\n",
+    "            container_class=dict\n",
+    "        )\n",
+    "\n",
+    "    def __iter__(self) -> Iterator[str]:\n",
+    "        for step, value in self.step_store.items():\n",
+    "            yield step.key\n",
+    "\n",
+    "    def __delitem__(self, key: str) -> None:\n",
+    "        step = GetItem(item=key)\n",
+    "\n",
+    "        if not step in self.step_store:\n",
+    "            raise ValueError(f\"Key not in mapped dictionary: {step}\")\n",
+    "\n",
+    "        val = self.step_store[step]\n",
+    "        del val\n",
+    "        del self.step_store[step]\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 142,
    "metadata": {},
    "outputs": [],
    "source": [
-    "temp = LensMappedDict()"
+    "temp = MappedContainer({\"hello\": 1, \"true\": [1,2,3]})"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 143,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{}"
+      ]
+     },
+     "execution_count": 143,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "temp.global_store"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 144,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{}"
+      ]
+     },
+     "execution_count": 144,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "temp['something'] = {'a': 1, 'b': 2, 'sub': {'c': 3, 'd': 4}}\n",
-    "temp.store"
+    "temp.global_store"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 145,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'hello': 1, 'true': [1, 2, 3], 'something': {'a': 1, 'b': 2, 'sub': {'c': 3, 'd': 4}}}"
+      ]
+     },
+     "execution_count": 145,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "temp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 146,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'hello': 1, 'true': [1, 2, 3], 'something': {'a': 1, 'b': 2, 'sub': 42}}"
+      ]
+     },
+     "execution_count": 146,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "temp['something']['sub'] = 42\n",
-    "temp.store"
+    "temp"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 147,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{Lens()['something']['sub']: 42}"
+      ]
+     },
+     "execution_count": 147,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
-    "type(temp.relative_store['something'])"
+    "temp.global_store"
    ]
   },
   {
@@ -228,7 +437,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
diff --git a/trulens_eval/trulens_eval/trace/__init__.py b/trulens_eval/trulens_eval/trace/__init__.py
index f578bab8d..82ce5e9a6 100644
--- a/trulens_eval/trulens_eval/trace/__init__.py
+++ b/trulens_eval/trulens_eval/trace/__init__.py
@@ -90,7 +90,9 @@ def serialize_contextmapping(
 """Type annotation for pydantic fields that store dictionaries whose keys are
 HashableSpanContext.
 
-This is needed to help pydantic figure out how to serialize and deserialize these dicts.
+This is needed to help pydantic figure out how to serialize and deserialize
+these dicts. SpanContext contains some non-hashable parts hence we convert these
+mappings to/from lists of tuples.
 """
 
 
@@ -110,7 +112,7 @@ class OTSpan(pydantic.BaseModel, ot_span.Span):
     See also [OpenTelemetry Span](https://opentelemetry.io/docs/specs/otel/trace/api/#span).
     """
 
-    _vendor: ClassVar[str] = "trulens_eval"
+    _vendor: ClassVar[str] = "tle" # "trulens_eval"
     """Vendor name as per OpenTelemetry attribute keys specifications."""
 
     @classmethod
diff --git a/trulens_eval/trulens_eval/trace/span.py b/trulens_eval/trulens_eval/trace/span.py
index 1759e33cd..8ef0bd42a 100644
--- a/trulens_eval/trulens_eval/trace/span.py
+++ b/trulens_eval/trulens_eval/trace/span.py
@@ -28,9 +28,11 @@
 T = TypeVar("T")
 
 class Span(mod_trace.OTSpan):
-    """Base Span type.
-    
-    Smallest unit of recorded activity.
+    """Base Span type for trulens_eval additions over base Open Telemetry spans.
+
+    Note that though this class and subclasses contain a variety of attributes
+    for data, everything is stored in the underlying Open Telemetry attributes/links
+    The attributes here map to/from Open Telemetry span attributes/links.
     """
 
     @staticmethod
@@ -61,7 +63,10 @@ def attribute_property(
                 of the property. This can be used for defaults that make use of
                 forward referenced types.
         """
+        # Remember whether we already called `initialize`.
         initialized = False
+
+        # pydantic type validator to be filled in later.
         tadapter = None
 
         def initialize():
@@ -69,12 +74,11 @@ def initialize():
             # property is used as otherwise forward references might not be
             # ready.
 
-            nonlocal initialized, tadapter
+            nonlocal initialized, tadapter, typ, default
 
             if initialized:
                 return
 
-            nonlocal typ, default
             if typ is None and typ_factory is not None:
                 typ = typ_factory()
 
@@ -87,15 +91,20 @@ def initialize():
             if typ is None:
                 tadapter = None
             else:
+                # Get the pydantic type checker and run it on the default value.
                 tadapter = TypeAdapter(typ)
                 if default is not None:
                     tadapter.validate_python(default)
 
         def getter(self) -> T:
+            nonlocal default
+
             initialize()
             return self.attributes.get(self.vendor_attr(name), default)
 
         def setter(self, value: T) -> None:
+            nonlocal tadapter
+
             initialize()
             if tadapter is not None:
                 tadapter.validate_python(value)
@@ -110,7 +119,6 @@ def setter(self, value: T) -> None:
     def start_datetime(self) -> datetime.datetime:
         """Start time of span as a [datetime][datetime.datetime]."""
         return mod_container_utils.datetime_of_ns_timestamp(self.start_timestamp)
-    
     @start_datetime.setter
     def start_datetime(self, value: datetime.datetime):
         self.start_timestamp = mod_container_utils.ns_timestamp_of_datetime(value)
@@ -119,7 +127,6 @@ def start_datetime(self, value: datetime.datetime):
     def end_datetime(self) -> datetime.datetime:
         """End time of span as a [datetime][datetime.datetime]."""
         return mod_container_utils.datetime_of_ns_timestamp(self.end_timestamp)
-    
     @end_datetime.setter
     def end_datetime(self, value: datetime.datetime):
         self.end_timestamp = mod_container_utils.ns_timestamp_of_datetime(value)
@@ -141,7 +148,7 @@ def parent_context(self) -> Optional[mod_trace.HashableSpanContext]:
         """Context of parent span if any.
 
         This is stored in OT links with a relationship attribute of "parent".
-        None if this is a root span or otherwise it does not have a parent.
+        None if this is a root span or otherwise does not have a parent.
         """
 
         for link_context, link_attributes in self.links.items():
@@ -188,8 +195,8 @@ def parent_span_id(self) -> Optional[mod_trace.TSpanID]:
     # will be set as a DictNamespace indexing elements in attributes
     @property
     def metadata(self) -> mod_container_utils.DictNamespace[ot_types.AttributeValue]:
+        """Metadata regarding a span."""
         return self.attributes_metadata
-
     @metadata.setter
     def metadata(self, value: Dict[str, str]):
         for k, v in value.items():
@@ -244,7 +251,6 @@ class SpanMethodCall(TransSpanRecord):
     error = Span.attribute_property("error", typ=Optional[Any], default_factory=None)
     # TODO: Need to encode to OT AttributeValue
 
-
 class TransSpanRecordAppCall(SpanMethodCall):
     """A Span which corresponds to single
     [RecordAppCall][trulens_eval.schema.record.RecordAppCall].
@@ -404,7 +410,7 @@ def to_class(self) -> Type[Span]:
 
         if hasattr(mod_trace.span, self.value):
             return getattr(mod_trace.span, self.value)
-        
+
         raise ValueError(f"Span type {self.value} not found in module.")
 
     UNTYPED = SpanUntyped.__name__
diff --git a/trulens_eval/trulens_eval/utils/serial.py b/trulens_eval/trulens_eval/utils/serial.py
index 80576f116..7208fb1fc 100644
--- a/trulens_eval/trulens_eval/utils/serial.py
+++ b/trulens_eval/trulens_eval/utils/serial.py
@@ -1031,8 +1031,10 @@ def _append(self, step: Step) -> Lens:
         return Lens(path=self.path + (step,))
 
     def __getitem__(
-        self, item: Union[int, str, slice, Sequence[int], Sequence[str]]
+        self, item: Union[int, str, slice, Sequence[int], Sequence[str], Step]
     ) -> 'Lens':
+        if isinstance(item, Step):
+            return self._append(item)
         if isinstance(item, int):
             return self._append(GetIndex(index=item))
         if isinstance(item, str):
@@ -1063,8 +1065,8 @@ def __getattr__(self, attr: str) -> 'Lens':
             # overwritten and it will not try to use any of the _repr_*_ methods
             # to display the object. In our case, this will result Lenses being
             # constructed with this canary attribute name. We instead return
-            # None here to let ipython know we have overwritten __getattr__ but
-            # we do not construct any Lenses.
+            # something here to let ipython know we have overwritten __getattr__
+            # but we do not construct any Lenses.
             return 0xdead
 
         return self._append(GetItemOrAttribute(item_or_attribute=attr))

From 80b9ab776a26220daf50ef7e827cab77ed6d7649 Mon Sep 17 00:00:00 2001
From: Piotr Mardziel <piotrm@gmail.com>
Date: Wed, 1 May 2024 18:27:18 -0700
Subject: [PATCH 3/4] working on mapping span attributes to otel attributes

---
 .../examples/experimental/spans_example.ipynb | 202 ++++++++++++------
 1 file changed, 133 insertions(+), 69 deletions(-)

diff --git a/trulens_eval/examples/experimental/spans_example.ipynb b/trulens_eval/examples/experimental/spans_example.ipynb
index 7f9b2eea5..0ca1c11b4 100644
--- a/trulens_eval/examples/experimental/spans_example.ipynb
+++ b/trulens_eval/examples/experimental/spans_example.ipynb
@@ -54,7 +54,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 141,
+   "execution_count": 155,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -83,23 +83,27 @@
     "T = TypeVar(\"T\")\n",
     "C = TypeVar(\"C\")\n",
     "\n",
-    "ValueOrContainer = Union[T, 'MappedContainer[T, C]']\n",
+    "ValueOrMappedContainer = Union[T, 'MappedContainer[T]']\n",
+    "ValueOrContainer = Union[T, C]\n",
     "\n",
-    "class MappedContainer(Generic[T, C]): # will behave like C[T]\n",
+    "def Mapped(value: T) -> ValueOrMappedContainer[T]:\n",
+    "    return MappedContainer.make_mapped_container(value=value, lens=Lens(), global_store={})\n",
+    "\n",
+    "class MappedContainer(Generic[T]):\n",
     "    def __init__(\n",
     "        self,\n",
-    "        value: Optional[C] = None,\n",
-    "        container_class: Optional[type] = None,\n",
+    "        value: Optional[ValueOrContainer[T, C]] = None,\n",
+    "        container_class: Optional[Type[C]] = None,\n",
     "        global_store: Optional[Dict[Lens, ot_types.AttributeValue]] = None,\n",
-    "        lens: Lens = None,\n",
+    "        lens: Optional[Lens] = None,\n",
     "    ):\n",
     "\n",
     "        if global_store is None:\n",
-    "            global_store: Dict[Lens, ot_types.AttributeValue] = {}\n",
+    "            global_store = {}\n",
     "\n",
     "        self.global_store = global_store\n",
     "\n",
-    "        self.step_store: Dict[Step, ValueOrContainer[T]] = {}\n",
+    "        self.step_store: Dict[Step, ValueOrMappedContainer[T]] = {}\n",
     "\n",
     "        if lens is None:\n",
     "            lens = Lens() \n",
@@ -111,22 +115,26 @@
     "            value = container_class()\n",
     "\n",
     "        if container_class is None:\n",
-    "            container_class = type(value)\n",
-    "        \n",
+    "            if isinstance(value, TPyJSONBase):\n",
+    "                container_class = None\n",
+    "            else:\n",
+    "                container_class = type(value)\n",
+    "\n",
     "        self.container_class = container_class\n",
     "\n",
-    "        self.map(value)\n",
+    "        if value is not None:\n",
+    "            self.map(value)\n",
     "\n",
     "    @staticmethod\n",
     "    def make_mapped_container(\n",
-    "        value: Optional[C] = None,\n",
-    "        container_class: Optional[type] = None,\n",
-    "        global_store: Optional[Dict[Lens, ot_types.AttributeValue]] = None,\n",
-    "        lens: Lens = None\n",
-    "    ) -> ValueOrContainer[T]:\n",
+    "        value: T,\n",
+    "        global_store: Dict[Lens, ot_types.AttributeValue],\n",
+    "        lens: Lens\n",
+    "    ) -> ValueOrMappedContainer[T]:\n",
     "        mtype = MappedContainer.mappedtype_for_value(value)\n",
     "\n",
     "        if mtype is None:\n",
+    "            global_store[lens] = value\n",
     "            return value\n",
     "        \n",
     "        return mtype(\n",
@@ -136,7 +144,7 @@
     "        )\n",
     "\n",
     "    @staticmethod\n",
-    "    def mappedtype_for_value(value: T) -> Optional[Type[MappedContainer[T, C]]]:\n",
+    "    def mappedtype_for_value(value: T) -> Optional[Type['MappedContainer[T]']]:\n",
     "        if isinstance(value, Mapping):\n",
     "            return MappedDict\n",
     "        elif isinstance(value, Sequence):\n",
@@ -154,6 +162,8 @@
     "        return repr(self.unmap())\n",
     "\n",
     "    def map(self, value: T, lens: Optional[Lens] = None) -> None:\n",
+    "        # print(f\"Mapping value under {self.lens}\", value, lens)\n",
+    "\n",
     "        if lens is None:\n",
     "            lens = self.lens\n",
     "\n",
@@ -183,42 +193,39 @@
     "        else:\n",
     "            raise TypeError(f\"Unexpected type: {type(value)}\")\n",
     "\n",
-    "        return self\n",
-    "\n",
-    "    def __getitem__(self, key: str) -> T:\n",
+    "    def __getitem__(self, step: Step) -> Optional[ValueOrMappedContainer[T]]:\n",
     "        if self.container_class is None:\n",
     "            raise TypeError(\"Cannot get item on a non-container.\")\n",
     "\n",
-    "        if self.container_class is dict:\n",
-    "            step = GetItem(item=key)\n",
-    "        elif self.container_class is list:\n",
-    "            step = GetIndex(index=key)\n",
-    "        else:\n",
-    "            raise TypeError(\"Unexpected container type.\")\n",
-    "\n",
     "        return self.step_store.get(step)\n",
     "\n",
     "    def __del__(self) -> None:\n",
+    "        # print(\"Deleting container\", self.lens)\n",
+    "\n",
+    "        for step, val in list(self.step_store.items()):\n",
+    "            if isinstance(val, MappedContainer):\n",
+    "                val.__del__()\n",
+    "            else:\n",
+    "                lens = self.lens[step]\n",
+    "                del self.global_store[lens]\n",
+    "\n",
+    "            del self.step_store[step]\n",
+    "\n",
     "        if self.container_class is None:\n",
     "            del self.global_store[self.lens]\n",
-    "            return\n",
     "\n",
-    "        for step, val in self.step_store.items():\n",
-    "            del val\n",
+    "    #def __delitem__(self, step: Step) -> None:\n",
+    "    #        if step in self.step_store:\n",
+    "    #            del (self.step_store[step])\n",
+    "\n",
+    "    def __setitem__(self, step: Step, value: T) -> None:\n",
+    "        # print(\"Setting step\", step, value)\n",
     "\n",
-    "    def __setitem__(self, key: str, value: T) -> None:\n",
     "        if self.container_class is None:\n",
     "            raise TypeError(\"Cannot set item on a non-container.\")\n",
     "\n",
-    "        if self.container_class is dict:\n",
-    "            step = GetItem(item=key)\n",
-    "        elif self.container_class is list:\n",
-    "            step = GetIndex(index=key)\n",
-    "        else:\n",
-    "            raise TypeError(\"Unexpected container type.\")\n",
-    "\n",
     "        if step in self.step_store:\n",
-    "            del self[key]\n",
+    "            del self.step_store[step]\n",
     "\n",
     "        if isinstance(value, TPyJSONBase):\n",
     "            mapped_value = value\n",
@@ -236,16 +243,16 @@
     "\n",
     "        self.step_store[step] = mapped_value\n",
     "\n",
-    "    def map_value(self, value: T) -> ValueOrContainer[T]:\n",
+    "    def map_value(self, value: T) -> ValueOrMappedContainer[T]:\n",
     "        return MappedContainer.make_mapped_container(\n",
     "            global_store=self.global_store, value=value, lens=self.lens\n",
     "        )\n",
     "\n",
-    "    def unmap(self) -> T:\n",
+    "    def unmap(self) -> ValueOrContainer[T, C]:\n",
     "        return MappedContainer.unmap_value(self)\n",
     "\n",
     "    @staticmethod\n",
-    "    def unmap_value(value: ValueOrContainer[T]) -> T:\n",
+    "    def unmap_value(value: ValueOrMappedContainer[T]) -> ValueOrContainer[T, C]:\n",
     "        if isinstance(value, TPyJSONBase):\n",
     "            return value\n",
     "\n",
@@ -260,12 +267,12 @@
     "        else:\n",
     "            raise TypeError(f\"Unexpected type: {type(value)}\")\n",
     "\n",
-    "class MappedList(MappedContainer[T, List], List[T]):\n",
+    "class MappedList(MappedContainer[T], List[T]):\n",
     "    def __init__(\n",
     "        self,\n",
     "        value: Optional[List[T]] = None,\n",
     "        global_store: Optional[Dict[Lens, ot_types.AttributeValue]] = None,\n",
-    "        lens: Lens = None,\n",
+    "        lens: Optional[Lens] = None,\n",
     "    ):\n",
     "        super().__init__(\n",
     "            value=value,\n",
@@ -273,27 +280,48 @@
     "            lens=lens,\n",
     "            container_class=list\n",
     "        )\n",
+    "        self.step_store: Dict[GetIndex, ValueOrMappedContainer[T]]\n",
     "\n",
-    "    def __iter__(self) -> Iterator[T]:\n",
-    "        for step, value in self.step_store.items():\n",
+    "    def __getitem__(self, index: int) -> Optional[ValueOrMappedContainer[T]]:\n",
+    "        if not isinstance(index, int):\n",
+    "            raise TypeError(\"Expected int index.\")\n",
+    "        \n",
+    "        step = GetIndex(index=index)\n",
+    "\n",
+    "        return super().__getitem__(step)\n",
+    "\n",
+    "    def __setitem__(self, index: int, value: T) -> None:\n",
+    "        # print(\"List: Setting item\", index, value)\n",
+    "\n",
+    "        if not isinstance(index, int):\n",
+    "            raise TypeError(\"Expected integer index.\")\n",
+    "        \n",
+    "        step = GetIndex(index=index)\n",
+    "\n",
+    "        super().__setitem__(step, value)\n",
+    "\n",
+    "    def __iter__(self) -> Iterator[ValueOrMappedContainer[T]]:\n",
+    "        for _, value in self.step_store.items():\n",
     "            yield value\n",
     "\n",
-    "    def __delitem__(self, key: str) -> None:\n",
-    "        step = GetIndex(index=key)\n",
+    "    def __delitem__(self, index: int) -> None:\n",
+    "        #print(\"List: Deleting item\", index)\n",
+    "\n",
+    "        step = GetIndex(index=index)\n",
     "    \n",
     "        if not step in self.step_store:\n",
-    "            raise ValueError(f\"Index not in mapped list: {step}\")\n",
+    "            raise IndexError(f\"Index out of range: {index}\")\n",
     "\n",
     "        val = self.step_store[step]\n",
     "        del val\n",
     "        del self.step_store[step]\n",
     "\n",
-    "class MappedDict(MappedContainer[T, Dict], Dict[str, T]):\n",
+    "class MappedDict(MappedContainer[T], Dict[str, T]):\n",
     "    def __init__(\n",
     "        self,\n",
     "        value: Optional[Dict[str, T]] = None,\n",
     "        global_store: Optional[Dict[Lens, ot_types.AttributeValue]] = None,\n",
-    "        lens: Lens = None,\n",
+    "        lens: Optional[Lens] = None,\n",
     "    ):\n",
     "        super().__init__(\n",
     "            value=value,\n",
@@ -301,15 +329,35 @@
     "            lens=lens,\n",
     "            container_class=dict\n",
     "        )\n",
+    "        self.step_store: Dict[GetItem, ValueOrMappedContainer[T]]\n",
+    "\n",
+    "    def __getitem__(self, key: str) -> Optional[ValueOrMappedContainer[T]]:\n",
+    "        if not isinstance(key, str):\n",
+    "            raise TypeError(\"Expected string key.\")\n",
+    "        \n",
+    "        step = GetItem(item=key)\n",
+    "\n",
+    "        return super().__getitem__(step)\n",
+    "\n",
+    "    def __setitem__(self, key: str, value: T) -> None:\n",
+    "        # print(\"Dict: Setting item\", key, value)\n",
+    "        if not isinstance(key, str):\n",
+    "            raise TypeError(\"Expected string key.\")\n",
+    "\n",
+    "        step = GetItem(item=key)\n",
+    "\n",
+    "        super().__setitem__(step, value)\n",
     "\n",
     "    def __iter__(self) -> Iterator[str]:\n",
     "        for step, value in self.step_store.items():\n",
-    "            yield step.key\n",
+    "            yield step.item\n",
     "\n",
     "    def __delitem__(self, key: str) -> None:\n",
+    "        # print(\"Dict: Deleting item\", key)\n",
+    "\n",
     "        step = GetItem(item=key)\n",
     "\n",
-    "        if not step in self.step_store:\n",
+    "        if step not in self.step_store:\n",
     "            raise ValueError(f\"Key not in mapped dictionary: {step}\")\n",
     "\n",
     "        val = self.step_store[step]\n",
@@ -319,25 +367,28 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 142,
+   "execution_count": 156,
    "metadata": {},
    "outputs": [],
    "source": [
-    "temp = MappedContainer({\"hello\": 1, \"true\": [1,2,3]})"
+    "temp = Mapped({\"hello\": 1, \"true\": [1,2,3]})"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 143,
+   "execution_count": 157,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "{}"
+       "{Lens()['hello']: 1,\n",
+       " Lens()['true'][0]: 1,\n",
+       " Lens()['true'][1]: 2,\n",
+       " Lens()['true'][2]: 3}"
       ]
      },
-     "execution_count": 143,
+     "execution_count": 157,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -348,16 +399,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 144,
+   "execution_count": 158,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "{}"
+       "{Lens()['hello']: 1,\n",
+       " Lens()['true'][0]: 1,\n",
+       " Lens()['true'][1]: 2,\n",
+       " Lens()['true'][2]: 3,\n",
+       " Lens()['something']['a']: 1,\n",
+       " Lens()['something']['b']: 2,\n",
+       " Lens()['something']['sub']['c']: 3,\n",
+       " Lens()['something']['sub']['d']: 4}"
       ]
      },
-     "execution_count": 144,
+     "execution_count": 158,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -369,7 +427,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 145,
+   "execution_count": 159,
    "metadata": {},
    "outputs": [
     {
@@ -378,7 +436,7 @@
        "{'hello': 1, 'true': [1, 2, 3], 'something': {'a': 1, 'b': 2, 'sub': {'c': 3, 'd': 4}}}"
       ]
      },
-     "execution_count": 145,
+     "execution_count": 159,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -389,7 +447,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 146,
+   "execution_count": 160,
    "metadata": {},
    "outputs": [
     {
@@ -398,7 +456,7 @@
        "{'hello': 1, 'true': [1, 2, 3], 'something': {'a': 1, 'b': 2, 'sub': 42}}"
       ]
      },
-     "execution_count": 146,
+     "execution_count": 160,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -410,16 +468,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 147,
+   "execution_count": 161,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "{Lens()['something']['sub']: 42}"
+       "{Lens()['hello']: 1,\n",
+       " Lens()['true'][0]: 1,\n",
+       " Lens()['true'][1]: 2,\n",
+       " Lens()['true'][2]: 3,\n",
+       " Lens()['something']['a']: 1,\n",
+       " Lens()['something']['b']: 2,\n",
+       " Lens()['something']['sub']: 42}"
       ]
      },
-     "execution_count": 147,
+     "execution_count": 161,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -437,7 +501,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [

From bb8808bf101b329ac6afbfe185ac128a0e64ce12 Mon Sep 17 00:00:00 2001
From: Piotr Mardziel <piotrm@gmail.com>
Date: Fri, 3 May 2024 10:59:55 -0700
Subject: [PATCH 4/4] clear outputs

---
 .../examples/experimental/spans_example.ipynb | 211 +++++-------------
 .../quickstart/langchain_quickstart.ipynb     | 162 +++++++++++++-
 trulens_eval/trulens_eval/trace/__init__.py   |   2 +-
 3 files changed, 213 insertions(+), 162 deletions(-)

diff --git a/trulens_eval/examples/experimental/spans_example.ipynb b/trulens_eval/examples/experimental/spans_example.ipynb
index 0ca1c11b4..b88a8727d 100644
--- a/trulens_eval/examples/experimental/spans_example.ipynb
+++ b/trulens_eval/examples/experimental/spans_example.ipynb
@@ -5,58 +5,6 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": [
-    "def lense_json(\n",
-    "    val: TPyJSON, lens: Lens = None\n",
-    ") -> Dict[Lens, ot_types.AttributeValue]:\n",
-    "    \"\"\"Convert a json structure to an OpenTelemetry attributes dict.\n",
-    "\n",
-    "    OT dicts have limited types in their values so a single JSON may need to be\n",
-    "    converted to multiple OT key/value pairs. The approach is to store\n",
-    "    paths/lenses into basic JSON elements, serialized into strings, as keys and\n",
-    "    those basic elements as values.\n",
-    "    \"\"\"\n",
-    "\n",
-    "    if lens is None:\n",
-    "        lens = Lens()\n",
-    "\n",
-    "    if isinstance(val, TPyJSONBase):\n",
-    "        return {lens: val}\n",
-    "\n",
-    "    if isinstance(val, Dict):\n",
-    "        ret = {}\n",
-    "        for k, v in val.items():\n",
-    "            sublens = lens[k]\n",
-    "            ret.update(lense_json(v, sublens))\n",
-    "        return ret\n",
-    "\n",
-    "    elif isinstance(val, Sequence):\n",
-    "        if len(val) == 0:\n",
-    "                return {lens: val}\n",
-    "        typ = type(val[0])\n",
-    "\n",
-    "        if isinstance(val[0], TPyJSON) and all(isinstance(x, typ) for x in val):\n",
-    "            return {lens: val}\n",
-    "        \n",
-    "        ret = {}\n",
-    "        for i in range(len(val)):\n",
-    "            sublens = lens[i]\n",
-    "            subval = val[i]\n",
-    "            ret.update(lense_json(subval, sublens))\n",
-    "\n",
-    "        return ret\n",
-    "\n",
-    "    else:\n",
-    "        raise ValueError(f\"Unexpected type: {type(val)}\")\n",
-    "\n",
-    "lense_json({\"hello\": 1, \"true\": 2})"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 155,
-   "metadata": {},
-   "outputs": [],
    "source": [
     "from typing import Union, Dict, List, Mapping, Sequence, TypeVar, Generic, Optional, Type, Iterator\n",
     "\n",
@@ -87,16 +35,18 @@
     "ValueOrContainer = Union[T, C]\n",
     "\n",
     "def Mapped(value: T) -> ValueOrMappedContainer[T]:\n",
-    "    return MappedContainer.make_mapped_container(value=value, lens=Lens(), global_store={})\n",
+    "    return MappedContainer.make_mapped_container(value=value, lens=Lens(), global_store={}, prefix=\"trulens_eval@\")\n",
     "\n",
     "class MappedContainer(Generic[T]):\n",
     "    def __init__(\n",
     "        self,\n",
     "        value: Optional[ValueOrContainer[T, C]] = None,\n",
     "        container_class: Optional[Type[C]] = None,\n",
-    "        global_store: Optional[Dict[Lens, ot_types.AttributeValue]] = None,\n",
+    "        global_store: Optional[Dict[str, ot_types.AttributeValue]] = None,\n",
     "        lens: Optional[Lens] = None,\n",
+    "        prefix: str = \"\",\n",
     "    ):\n",
+    "        self.prefix = prefix\n",
     "\n",
     "        if global_store is None:\n",
     "            global_store = {}\n",
@@ -125,22 +75,33 @@
     "        if value is not None:\n",
     "            self.map(value)\n",
     "\n",
+    "    def _global_key(self, lens: Lens) -> str:\n",
+    "        return self._make_global_key(lens, self.prefix)\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def _make_global_key(lens: Lens, prefix: str = \"\") -> str:\n",
+    "        return prefix + str(lens)\n",
+    "\n",
     "    @staticmethod\n",
     "    def make_mapped_container(\n",
     "        value: T,\n",
-    "        global_store: Dict[Lens, ot_types.AttributeValue],\n",
-    "        lens: Lens\n",
+    "        global_store: Dict[str, ot_types.AttributeValue],\n",
+    "        lens: Lens, \n",
+    "        prefix: str = \"\"\n",
     "    ) -> ValueOrMappedContainer[T]:\n",
     "        mtype = MappedContainer.mappedtype_for_value(value)\n",
     "\n",
     "        if mtype is None:\n",
-    "            global_store[lens] = value\n",
+    "            global_store[\n",
+    "                MappedContainer._make_global_key(lens, prefix)\n",
+    "            ] = value\n",
     "            return value\n",
     "        \n",
     "        return mtype(\n",
     "            global_store=global_store,\n",
     "            value=value,\n",
-    "            lens=lens\n",
+    "            lens=lens,\n",
+    "            prefix=prefix\n",
     "        )\n",
     "\n",
     "    @staticmethod\n",
@@ -168,7 +129,7 @@
     "            lens = self.lens\n",
     "\n",
     "        if isinstance(value, TPyJSONBase):\n",
-    "            self.global_store[lens] = value\n",
+    "            self.global_store[self._global_key(lens)] = value\n",
     "\n",
     "        elif isinstance(value, Sequence):\n",
     "            for i in range(len(value)):\n",
@@ -176,7 +137,8 @@
     "                subcontainer = MappedContainer.make_mapped_container(\n",
     "                    global_store=self.global_store,\n",
     "                    lens=self.lens[step],\n",
-    "                    value=value[i]\n",
+    "                    value=value[i],\n",
+    "                    prefix=self.prefix,\n",
     "                )\n",
     "                self.step_store[step] = subcontainer\n",
     "\n",
@@ -186,7 +148,8 @@
     "                subcontainer = MappedContainer.make_mapped_container(\n",
     "                    global_store=self.global_store,\n",
     "                    lens=self.lens[step],\n",
-    "                    value=v\n",
+    "                    value=v,\n",
+    "                    prefix=self.prefix\n",
     "                )\n",
     "                self.step_store[step] = subcontainer\n",
     "\n",
@@ -207,16 +170,12 @@
     "                val.__del__()\n",
     "            else:\n",
     "                lens = self.lens[step]\n",
-    "                del self.global_store[lens]\n",
+    "                del self.global_store[self._global_key(lens)]\n",
     "\n",
     "            del self.step_store[step]\n",
     "\n",
     "        if self.container_class is None:\n",
-    "            del self.global_store[self.lens]\n",
-    "\n",
-    "    #def __delitem__(self, step: Step) -> None:\n",
-    "    #        if step in self.step_store:\n",
-    "    #            del (self.step_store[step])\n",
+    "            del self.global_store[self._global_key(self.lens)]\n",
     "\n",
     "    def __setitem__(self, step: Step, value: T) -> None:\n",
     "        # print(\"Setting step\", step, value)\n",
@@ -229,13 +188,14 @@
     "\n",
     "        if isinstance(value, TPyJSONBase):\n",
     "            mapped_value = value\n",
-    "            self.global_store[self.lens[step]] = value\n",
+    "            self.global_store[self._global_key(self.lens[step])] = value\n",
     "\n",
     "        elif isinstance(value, (Mapping, Sequence)):\n",
     "            mapped_value = MappedContainer.make_mapped_container(\n",
     "                global_store=self.global_store,\n",
     "                value=value,\n",
-    "                lens=self.lens[step]\n",
+    "                lens=self.lens[step],\n",
+    "                prefix=self.prefix\n",
     "            )\n",
     "\n",
     "        else:\n",
@@ -245,7 +205,7 @@
     "\n",
     "    def map_value(self, value: T) -> ValueOrMappedContainer[T]:\n",
     "        return MappedContainer.make_mapped_container(\n",
-    "            global_store=self.global_store, value=value, lens=self.lens\n",
+    "            global_store=self.global_store, value=value, lens=self.lens, prefix=self.prefix\n",
     "        )\n",
     "\n",
     "    def unmap(self) -> ValueOrContainer[T, C]:\n",
@@ -258,7 +218,7 @@
     "\n",
     "        if isinstance(value, MappedContainer):\n",
     "            if value.container_class is None:\n",
-    "                return value.global_store[value.lens]\n",
+    "                return value.global_store[self._global_key(value.lens)]\n",
     "\n",
     "            container = value.container_class()\n",
     "            for step, v in value.step_store.items():\n",
@@ -271,14 +231,16 @@
     "    def __init__(\n",
     "        self,\n",
     "        value: Optional[List[T]] = None,\n",
-    "        global_store: Optional[Dict[Lens, ot_types.AttributeValue]] = None,\n",
+    "        global_store: Optional[Dict[str, ot_types.AttributeValue]] = None,\n",
     "        lens: Optional[Lens] = None,\n",
+    "        prefix: str = \"\"\n",
     "    ):\n",
     "        super().__init__(\n",
     "            value=value,\n",
     "            global_store=global_store,\n",
     "            lens=lens,\n",
-    "            container_class=list\n",
+    "            container_class=list,\n",
+    "            prefix=prefix\n",
     "        )\n",
     "        self.step_store: Dict[GetIndex, ValueOrMappedContainer[T]]\n",
     "\n",
@@ -320,14 +282,16 @@
     "    def __init__(\n",
     "        self,\n",
     "        value: Optional[Dict[str, T]] = None,\n",
-    "        global_store: Optional[Dict[Lens, ot_types.AttributeValue]] = None,\n",
+    "        global_store: Optional[Dict[str, ot_types.AttributeValue]] = None,\n",
     "        lens: Optional[Lens] = None,\n",
+    "        prefix: str = \"\"\n",
     "    ):\n",
     "        super().__init__(\n",
     "            value=value,\n",
     "            global_store=global_store,\n",
     "            lens=lens,\n",
-    "            container_class=dict\n",
+    "            container_class=dict,\n",
+    "            prefix=prefix\n",
     "        )\n",
     "        self.step_store: Dict[GetItem, ValueOrMappedContainer[T]]\n",
     "\n",
@@ -367,7 +331,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 156,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -376,50 +340,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 157,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{Lens()['hello']: 1,\n",
-       " Lens()['true'][0]: 1,\n",
-       " Lens()['true'][1]: 2,\n",
-       " Lens()['true'][2]: 3}"
-      ]
-     },
-     "execution_count": 157,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "temp.global_store"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 158,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{Lens()['hello']: 1,\n",
-       " Lens()['true'][0]: 1,\n",
-       " Lens()['true'][1]: 2,\n",
-       " Lens()['true'][2]: 3,\n",
-       " Lens()['something']['a']: 1,\n",
-       " Lens()['something']['b']: 2,\n",
-       " Lens()['something']['sub']['c']: 3,\n",
-       " Lens()['something']['sub']['d']: 4}"
-      ]
-     },
-     "execution_count": 158,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "temp['something'] = {'a': 1, 'b': 2, 'sub': {'c': 3, 'd': 4}}\n",
     "temp.global_store"
@@ -427,40 +359,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 159,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'hello': 1, 'true': [1, 2, 3], 'something': {'a': 1, 'b': 2, 'sub': {'c': 3, 'd': 4}}}"
-      ]
-     },
-     "execution_count": 159,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "temp"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 160,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'hello': 1, 'true': [1, 2, 3], 'something': {'a': 1, 'b': 2, 'sub': 42}}"
-      ]
-     },
-     "execution_count": 160,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "temp['something']['sub'] = 42\n",
     "temp"
@@ -468,28 +378,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 161,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{Lens()['hello']: 1,\n",
-       " Lens()['true'][0]: 1,\n",
-       " Lens()['true'][1]: 2,\n",
-       " Lens()['true'][2]: 3,\n",
-       " Lens()['something']['a']: 1,\n",
-       " Lens()['something']['b']: 2,\n",
-       " Lens()['something']['sub']: 42}"
-      ]
-     },
-     "execution_count": 161,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "temp.global_store"
+    "temp"
    ]
   },
   {
@@ -501,7 +394,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
diff --git a/trulens_eval/examples/quickstart/langchain_quickstart.ipynb b/trulens_eval/examples/quickstart/langchain_quickstart.ipynb
index e601a0745..93f6c0224 100644
--- a/trulens_eval/examples/quickstart/langchain_quickstart.ipynb
+++ b/trulens_eval/examples/quickstart/langchain_quickstart.ipynb
@@ -1,5 +1,60 @@
 {
  "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# pip uninstall -y trulens_eval\n",
+    "# pip install git+https://github.com/truera/trulens@piotrm/azure_bugfixes#subdirectory=trulens_eval\n",
+    "\n",
+    "# trulens_eval notebook dev\n",
+    "\n",
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "from pathlib import Path\n",
+    "import sys\n",
+    "\n",
+    "base = Path().cwd()\n",
+    "while not (base / \"trulens_eval\" / \"trulens_eval\").exists():\n",
+    "    base = base.parent\n",
+    "\n",
+    "if (base / \"trulens_eval\" / \"trulens_eval\").exists():\n",
+    "    base = base / \"trulens_eval\"\n",
+    "\n",
+    "print(base)\n",
+    "\n",
+    "# If running from github repo, can use this:\n",
+    "sys.path.append(str(base))\n",
+    "\n",
+    "# Uncomment for more debugging printouts.\n",
+    "\"\"\"\n",
+    "import logging\n",
+    "root = logging.getLogger()\n",
+    "root.setLevel(logging.DEBUG)\n",
+    "\n",
+    "handler = logging.StreamHandler(sys.stdout)\n",
+    "handler.setLevel(logging.DEBUG)\n",
+    "formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')\n",
+    "handler.setFormatter(formatter)\n",
+    "root.addHandler(handler)\n",
+    "\"\"\"\n",
+    "\n",
+    "from trulens_eval.keys import check_keys\n",
+    "\n",
+    "check_keys(\n",
+    "    \"OPENAI_API_KEY\",\n",
+    "    \"HUGGINGFACE_API_KEY\"\n",
+    ")\n",
+    "\n",
+    "from trulens_eval import Tru\n",
+    "tru = Tru()\n",
+    "tru.reset_database()\n",
+    "\n",
+    "tru.run_dashboard(_dev=base, force=True)"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",
@@ -169,6 +224,101 @@
     "rag_chain.invoke(\"What is Task Decomposition?\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rag_chain"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_core.runnables import RunnablePassthrough, RunnableSequence\n",
+    "\n",
+    "# how we mark a method as instrumented\n",
+    "print(rag_chain.invoke.__tru_instrumented)\n",
+    "\n",
+    "# a method may have multiple imlementations in different classes across the MRO\n",
+    "# \"method resolution order\"\n",
+    "for superclass in rag_chain.__class__.__mro__:\n",
+    "    print(superclass.__name__, superclass.__module__, end=\"\")\n",
+    "    if hasattr(superclass, \"ainvoke\") and hasattr(superclass.ainvoke, \"__tru_instrumented\"):\n",
+    "        print(\" is instrumented\")\n",
+    "    else:\n",
+    "        print()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# rag_chain.invoke != RunnableSequence.invoke\n",
+    "# rag_chain.invoke is \"bound\" in that it has a value for \"self\", whereas\n",
+    "# RunnableSequence.invoke expects self as an argument.\n",
+    "print(RunnableSequence.invoke == rag_chain.invoke)                # false\n",
+    "print(RunnableSequence.invoke == rag_chain.invoke.__func__)       # true\n",
+    "print(RunnablePassthrough.invoke == rag_chain.invoke.__func__)    # false\n",
+    "print(rag_chain.invoke.__self__)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Look at one inner component, a retriever:\n",
+    "print(type(rag_chain.steps[0].steps['context'].steps[0]))\n",
+    "# And its method of interest:\n",
+    "print(rag_chain.steps[0].steps['context'].steps[0].get_relevant_documents)\n",
+    "# Recall selector for context:\n",
+    "print(context)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from trulens_eval.utils.serial import Lens\n",
+    "context_part = Lens(path=context.path[2:-1])\n",
+    "print(context_part)\n",
+    "print(context_part.get_sole_item(rag_chain))\n",
+    "print(context_part.get_sole_item(rag_chain) == rag_chain.steps[0].steps['context'].steps[0].get_relevant_documents)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(type(rag_chain))\n",
+    "for attr in dir(rag_chain):\n",
+    "    if attr.startswith(\"_\"):\n",
+    "        continue\n",
+    "    try:\n",
+    "        print(attr, type(getattr(rag_chain, attr)))\n",
+    "    except Exception:\n",
+    "        pass"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",
@@ -243,7 +393,15 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "response, tru_record = tru_recorder.with_record(rag_chain.invoke, \"What is Task Decomposition?\")"
+    "response, tru_record = tru_recorder.with_record(\n",
+    "    rag_chain.invoke,\n",
+    "    \"What is Task Decomposition?\"\n",
+    ")\n",
+    "\n",
+    "with tru_recorder as recorder:\n",
+    "    rag_chain.invoke(\"What is Task Decomposition?\")\n",
+    "\n",
+    "tru_record = recorder.get()"
    ]
   },
   {
@@ -441,7 +599,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.18"
+   "version": "3.11.6"
   },
   "vscode": {
    "interpreter": {
diff --git a/trulens_eval/trulens_eval/trace/__init__.py b/trulens_eval/trulens_eval/trace/__init__.py
index 82ce5e9a6..a16dbc7ce 100644
--- a/trulens_eval/trulens_eval/trace/__init__.py
+++ b/trulens_eval/trulens_eval/trace/__init__.py
@@ -112,7 +112,7 @@ class OTSpan(pydantic.BaseModel, ot_span.Span):
     See also [OpenTelemetry Span](https://opentelemetry.io/docs/specs/otel/trace/api/#span).
     """
 
-    _vendor: ClassVar[str] = "tle" # "trulens_eval"
+    _vendor: ClassVar[str] = "trulens_eval"
     """Vendor name as per OpenTelemetry attribute keys specifications."""
 
     @classmethod