notebooks/llm_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6f1b0007-0e9d-4c63-9f8e-7091b658ce34",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install trulens-eval\n",
    "# !pip install pysqlite3\n",
    "# !pip install ipytree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6bba62bd-79bd-458d-8c68-872bc598ddd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../docker-lambda-aws/image/src/')\n",
    "\n",
    "import logging\n",
    "from ipytree import Tree, Node\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "import os\n",
    "import openai\n",
    "from trulens_eval import TruChain, Tru\n",
    "from trulens_eval.feedback.provider import OpenAI\n",
    "from trulens_eval import Feedback\n",
    "from trulens_eval.app import App\n",
    "from trulens_eval.feedback import Groundedness\n",
    "import numpy as np\n",
    "from langchain.schema.runnable import RunnableMap\n",
    "from langchain.schema.output_parser import StrOutputParser\n",
    "from langchain_openai import ChatOpenAI\n",
    "from langchain.prompts import ChatPromptTemplate\n",
    "from langchain_core.runnables import RunnablePassthrough\n",
    "\n",
    "from retriever import get_retriever\n",
    "import config\n",
    "from curriebot import get_agent_executor\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9c3f796d-b2f1-4ed6-995e-314947b08e92",
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.basicConfig()\n",
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3df6edd9-3730-45e7-94e3-597154984f76",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
      "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
      "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
      "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🦑 Tru initialized with db url sqlite:///default.sqlite .\n",
      "🛑 Secret keys may be written to the database. See the `database_redact_keys` option of Tru` to prevent this.\n"
     ]
    }
   ],
   "source": [
    "_ = load_dotenv(find_dotenv()) \n",
    "\n",
    "openai.api_key  = os.environ['OPENAI_API_KEY']\n",
    "\n",
    "tru = Tru()\n",
    "tru.reset_database()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a2b67dd-9f26-4ad2-8ce3-ff8918a494dd",
   "metadata": {},
   "source": [
    "# Utility functions "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6012f19-29b7-4219-90dc-44794cdf1350",
   "metadata": {},
   "source": [
    "Reference: https://www.trulens.org/trulens_eval/getting_started/quickstarts/langchain_quickstart/#load-documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e87b9dc2-0c36-4831-aa1b-35c83615e4d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_docs(docs):\n",
    "    return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
    "\n",
    "def display_call_stack(data):\n",
    "    tree = Tree()\n",
    "    tree.add_node(Node('Record ID: {}'.format(data['record_id'])))\n",
    "    tree.add_node(Node('App ID: {}'.format(data['app_id'])))\n",
    "    tree.add_node(Node('Cost: {}'.format(data['cost'])))\n",
    "    tree.add_node(Node('Performance: {}'.format(data['perf'])))\n",
    "    tree.add_node(Node('Timestamp: {}'.format(data['ts'])))\n",
    "    tree.add_node(Node('Tags: {}'.format(data['tags'])))\n",
    "    tree.add_node(Node('Main Input: {}'.format(data['main_input'])))\n",
    "    tree.add_node(Node('Main Output: {}'.format(data['main_output'])))\n",
    "    tree.add_node(Node('Main Error: {}'.format(data['main_error'])))\n",
    "    \n",
    "    calls_node = Node('Calls')\n",
    "    tree.add_node(calls_node)\n",
    "    \n",
    "    for call in data['calls']:\n",
    "        call_node = Node('Call')\n",
    "        calls_node.add_node(call_node)\n",
    "        \n",
    "        for step in call['stack']:\n",
    "            step_node = Node('Step: {}'.format(step['path']))\n",
    "            call_node.add_node(step_node)\n",
    "            if 'expanded' in step:\n",
    "                expanded_node = Node('Expanded')\n",
    "                step_node.add_node(expanded_node)\n",
    "                for expanded_step in step['expanded']:\n",
    "                    expanded_step_node = Node('Step: {}'.format(expanded_step['path']))\n",
    "                    expanded_node.add_node(expanded_step_node)\n",
    "    \n",
    "    return tree\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ec1e18f-00a9-4471-94bb-1088740c36c4",
   "metadata": {},
   "source": [
    "# Evaluate RAG "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6ce4e934-cd77-4256-8f84-04b43da6d842",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = ChatOpenAI(model=config.model_name, temperature=config.temperature)\n",
    "retriever = get_retriever()\n",
    "output_parser = StrOutputParser()\n",
    "retrieval_resume_template = ChatPromptTemplate.from_messages([\n",
    "    (\"system\", \"\"\"Your task is to answer questions about the resume provided.\n",
    "                  Answer the question based only on the following context.\n",
    "                  If you cannot answer the question or there are no resume context provided \n",
    "                  tell the user to message `https://www.linkedin.com/in/patricklarosa/`\n",
    "\n",
    "                  [RESUME CONTEXT]\n",
    "                  {context}\n",
    "                  \"\"\"),\n",
    "    (\"human\", \"{question}\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "07d7f38b-7b91-4b0d-a065-45ce8ac2d5eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "chain = (\n",
    "    {\"context\": retriever | format_docs, \"question\": RunnablePassthrough()}\n",
    "    | retrieval_resume_template\n",
    "    | llm\n",
    "    | StrOutputParser()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e252342c-d6d0-4653-b7bc-9166ef602c6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "provider = OpenAI()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "494907ce-3046-425f-bf96-5a92449ddde8",
   "metadata": {},
   "outputs": [],
   "source": [
    "context = App.select_context(chain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d0ee771f-7814-4bcc-bdc4-d63b88c77ceb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✅ In groundedness_measure_with_cot_reasons, input source will be set to __record__.app.first.steps__.context.first.get_relevant_documents.rets.collect() .\n",
      "✅ In groundedness_measure_with_cot_reasons, input statement will be set to __record__.main_output or `Select.RecordOutput` .\n",
      "✅ In relevance, input prompt will be set to __record__.main_input or `Select.RecordInput` .\n",
      "✅ In relevance, input response will be set to __record__.main_output or `Select.RecordOutput` .\n",
      "✅ In context_relevance_with_cot_reasons, input question will be set to __record__.main_input or `Select.RecordInput` .\n",
      "✅ In context_relevance_with_cot_reasons, input context will be set to __record__.app.first.steps__.context.first.get_relevant_documents.rets .\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/ubuntu/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "grounded = Groundedness(groundedness_provider=OpenAI())\n",
    "# Define a groundedness feedback function\n",
    "f_groundedness = (\n",
    "    Feedback(grounded.groundedness_measure_with_cot_reasons)\n",
    "    .on(context.collect()) # collect context chunks into a list\n",
    "    .on_output()\n",
    "    .aggregate(grounded.grounded_statements_aggregator)\n",
    ")\n",
    "\n",
    "# Question/answer relevance between overall question and answer.\n",
    "f_answer_relevance = (\n",
    "    Feedback(provider.relevance)\n",
    "    .on_input_output()\n",
    ")\n",
    "# Question/statement relevance between question and each context chunk.\n",
    "f_context_relevance = (\n",
    "    Feedback(provider.context_relevance_with_cot_reasons)\n",
    "    .on_input()\n",
    "    .on(context)\n",
    "    .aggregate(np.mean)\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "42dbbc9c-2f9e-4f24-8e4c-4fc1f746b697",
   "metadata": {},
   "outputs": [],
   "source": [
    "tru_recorder = TruChain(chain,\n",
    "    app_id='ResumeRetrieverApplication',\n",
    "    feedbacks=[f_answer_relevance, f_context_relevance, f_groundedness])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6436383d-d4c5-48ba-90de-07843f7e8d5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2414ec0e67574d2487409a215cefa43e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Groundedness per statement in source:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15c8190f24924425a4820a59d3237106",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Groundedness per statement in source:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a05d1a5382aa48ed9c2eb77c62d1cec5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Groundedness per statement in source:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50afc408b02c4cbe9c97cef486fa299b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Groundedness per statement in source:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b83310186f554e818202fc39bf8afd9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Groundedness per statement in source:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2824ced5eeba4d6b91138c55f519908c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Groundedness per statement in source:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66511c2a94224f4180d720da712a0cbc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Groundedness per statement in source:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a667769e9dfb492ba560212d86c8e958",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Groundedness per statement in source:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f84e41f01fbf4049a5748691e06d89a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Groundedness per statement in source:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with tru_recorder as recording:\n",
    "    llm_response = chain.invoke(\"Can you share his work experience?\")\n",
    "    llm_response = chain.invoke(\"Does he have experience with generative ai?\")\n",
    "    llm_response = chain.invoke(\"What is his latest employment?\")\n",
    "    llm_response = chain.invoke(\"What is his first job?\")\n",
    "    llm_response = chain.invoke(\"total years of experience?\")\n",
    "    llm_response = chain.invoke(\"What is his professional summary?\")\n",
    "    llm_response = chain.invoke(\"What is his highest educational attainment?\")\n",
    "    llm_response = chain.invoke(\"When is his birthday?\")\n",
    "    llm_response = chain.invoke(\"When is his mobile number?\")\n",
    "    llm_response = chain.invoke(\"When are his hobbies?\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ac5d323d-3c82-4594-a2ad-70cdfc9dbe56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app_id</th>\n",
       "      <th>app_json</th>\n",
       "      <th>type</th>\n",
       "      <th>record_id</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "      <th>tags</th>\n",
       "      <th>record_json</th>\n",
       "      <th>cost_json</th>\n",
       "      <th>perf_json</th>\n",
       "      <th>ts</th>\n",
       "      <th>relevance</th>\n",
       "      <th>context_relevance_with_cot_reasons</th>\n",
       "      <th>groundedness_measure_with_cot_reasons</th>\n",
       "      <th>relevance_calls</th>\n",
       "      <th>context_relevance_with_cot_reasons_calls</th>\n",
       "      <th>groundedness_measure_with_cot_reasons_calls</th>\n",
       "      <th>latency</th>\n",
       "      <th>total_tokens</th>\n",
       "      <th>total_cost</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ResumeRetrieverApplication</td>\n",
       "      <td>{\"tru_class_info\": {\"name\": \"TruChain\", \"modul...</td>\n",
       "      <td>RunnableSequence(langchain_core.runnables.base)</td>\n",
       "      <td>record_hash_b727a00d3855f32b55678a653a4a192f</td>\n",
       "      <td>\"Can you share his work experience?\"</td>\n",
       "      <td>\"Certainly! Here's a summary of the work exper...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_b727a00d3855f32b556...</td>\n",
       "      <td>{\"n_requests\": 3, \"n_successful_requests\": 3, ...</td>\n",
       "      <td>{\"start_time\": \"2024-05-14T14:22:28.171547\", \"...</td>\n",
       "      <td>2024-05-14T14:23:01.413212</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.653846</td>\n",
       "      <td>[{'args': {'prompt': 'Can you share his work e...</td>\n",
       "      <td>[{'args': {'question': 'Can you share his work...</td>\n",
       "      <td>[{'args': {'source': [[{'page_content': '- Rec...</td>\n",
       "      <td>33</td>\n",
       "      <td>2167</td>\n",
       "      <td>0.05141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ResumeRetrieverApplication</td>\n",
       "      <td>{\"tru_class_info\": {\"name\": \"TruChain\", \"modul...</td>\n",
       "      <td>RunnableSequence(langchain_core.runnables.base)</td>\n",
       "      <td>record_hash_bf8ca5f284718dbf6fc4f0caebc171ea</td>\n",
       "      <td>\"Does he have experience with generative ai?\"</td>\n",
       "      <td>\"Yes, he has experience with generative AI. He...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_bf8ca5f284718dbf6fc...</td>\n",
       "      <td>{\"n_requests\": 3, \"n_successful_requests\": 3, ...</td>\n",
       "      <td>{\"start_time\": \"2024-05-14T14:23:01.813168\", \"...</td>\n",
       "      <td>2024-05-14T14:23:11.056413</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>[{'args': {'prompt': 'Does he have experience ...</td>\n",
       "      <td>[{'args': {'question': 'Does he have experienc...</td>\n",
       "      <td>[{'args': {'source': [[{'page_content': '- Rec...</td>\n",
       "      <td>9</td>\n",
       "      <td>1815</td>\n",
       "      <td>0.04087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ResumeRetrieverApplication</td>\n",
       "      <td>{\"tru_class_info\": {\"name\": \"TruChain\", \"modul...</td>\n",
       "      <td>RunnableSequence(langchain_core.runnables.base)</td>\n",
       "      <td>record_hash_32117b90f955318c0a1d982c251850d8</td>\n",
       "      <td>\"What is his latest employment?\"</td>\n",
       "      <td>\"The latest employment is at TDCX, where the i...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_32117b90f955318c0a1...</td>\n",
       "      <td>{\"n_requests\": 3, \"n_successful_requests\": 3, ...</td>\n",
       "      <td>{\"start_time\": \"2024-05-14T14:23:11.475942\", \"...</td>\n",
       "      <td>2024-05-14T14:23:18.797011</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>[{'args': {'prompt': 'What is his latest emplo...</td>\n",
       "      <td>[{'args': {'question': 'What is his latest emp...</td>\n",
       "      <td>[{'args': {'source': [[{'page_content': '## Ca...</td>\n",
       "      <td>7</td>\n",
       "      <td>1753</td>\n",
       "      <td>0.03913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ResumeRetrieverApplication</td>\n",
       "      <td>{\"tru_class_info\": {\"name\": \"TruChain\", \"modul...</td>\n",
       "      <td>RunnableSequence(langchain_core.runnables.base)</td>\n",
       "      <td>record_hash_61030784c623d05423e7b2d1984d7240</td>\n",
       "      <td>\"What is his first job?\"</td>\n",
       "      <td>\"The first job mentioned in the resume is the ...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_61030784c623d05423e...</td>\n",
       "      <td>{\"n_requests\": 3, \"n_successful_requests\": 3, ...</td>\n",
       "      <td>{\"start_time\": \"2024-05-14T14:23:19.206538\", \"...</td>\n",
       "      <td>2024-05-14T14:23:24.755373</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>[{'args': {'prompt': 'What is his first job?',...</td>\n",
       "      <td>[{'args': {'question': 'What is his first job?...</td>\n",
       "      <td>[{'args': {'source': [[{'page_content': 'Busin...</td>\n",
       "      <td>5</td>\n",
       "      <td>1813</td>\n",
       "      <td>0.03901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ResumeRetrieverApplication</td>\n",
       "      <td>{\"tru_class_info\": {\"name\": \"TruChain\", \"modul...</td>\n",
       "      <td>RunnableSequence(langchain_core.runnables.base)</td>\n",
       "      <td>record_hash_60a8a754d52f9ae639533abd2bf4a5e0</td>\n",
       "      <td>\"total years of experience?\"</td>\n",
       "      <td>\"Based on the resume provided, Patrick La Rosa...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_60a8a754d52f9ae6395...</td>\n",
       "      <td>{\"n_requests\": 3, \"n_successful_requests\": 3, ...</td>\n",
       "      <td>{\"start_time\": \"2024-05-14T14:23:25.129293\", \"...</td>\n",
       "      <td>2024-05-14T14:23:30.275122</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>[{'args': {'prompt': 'total years of experienc...</td>\n",
       "      <td>[{'args': {'question': 'total years of experie...</td>\n",
       "      <td>[{'args': {'source': [[{'page_content': 'Busin...</td>\n",
       "      <td>5</td>\n",
       "      <td>1799</td>\n",
       "      <td>0.03886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ResumeRetrieverApplication</td>\n",
       "      <td>{\"tru_class_info\": {\"name\": \"TruChain\", \"modul...</td>\n",
       "      <td>RunnableSequence(langchain_core.runnables.base)</td>\n",
       "      <td>record_hash_9c817439e3e1d0a104d18d8b3fbd6ac8</td>\n",
       "      <td>\"What is his professional summary?\"</td>\n",
       "      <td>\"Patrick is an engineer by education, a data s...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_9c817439e3e1d0a104d...</td>\n",
       "      <td>{\"n_requests\": 3, \"n_successful_requests\": 3, ...</td>\n",
       "      <td>{\"start_time\": \"2024-05-14T14:23:30.589785\", \"...</td>\n",
       "      <td>2024-05-14T14:23:38.568031</td>\n",
       "      <td>0.9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>[{'args': {'prompt': 'What is his professional...</td>\n",
       "      <td>[{'args': {'question': 'What is his profession...</td>\n",
       "      <td>[{'args': {'source': [[{'page_content': '# PRO...</td>\n",
       "      <td>7</td>\n",
       "      <td>1281</td>\n",
       "      <td>0.03530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ResumeRetrieverApplication</td>\n",
       "      <td>{\"tru_class_info\": {\"name\": \"TruChain\", \"modul...</td>\n",
       "      <td>RunnableSequence(langchain_core.runnables.base)</td>\n",
       "      <td>record_hash_05c6af17e5e86415f8fd82ab7cbf5251</td>\n",
       "      <td>\"What is his highest educational attainment?\"</td>\n",
       "      <td>\"Patrick La Rosa's highest educational attainm...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_05c6af17e5e86415f8f...</td>\n",
       "      <td>{\"n_requests\": 3, \"n_successful_requests\": 3, ...</td>\n",
       "      <td>{\"start_time\": \"2024-05-14T14:23:38.892871\", \"...</td>\n",
       "      <td>2024-05-14T14:23:44.948560</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>[{'args': {'prompt': 'What is his highest educ...</td>\n",
       "      <td>[{'args': {'question': 'What is his highest ed...</td>\n",
       "      <td>[{'args': {'source': [[{'page_content': '# EDU...</td>\n",
       "      <td>6</td>\n",
       "      <td>1198</td>\n",
       "      <td>0.03313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ResumeRetrieverApplication</td>\n",
       "      <td>{\"tru_class_info\": {\"name\": \"TruChain\", \"modul...</td>\n",
       "      <td>RunnableSequence(langchain_core.runnables.base)</td>\n",
       "      <td>record_hash_7ccf30171368929b740781f2c79e7bb7</td>\n",
       "      <td>\"When is his birthday?\"</td>\n",
       "      <td>\"The resume does not provide information regar...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_7ccf30171368929b740...</td>\n",
       "      <td>{\"n_requests\": 3, \"n_successful_requests\": 3, ...</td>\n",
       "      <td>{\"start_time\": \"2024-05-14T14:23:45.360881\", \"...</td>\n",
       "      <td>2024-05-14T14:23:52.098814</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>[{'args': {'prompt': 'When is his birthday?', ...</td>\n",
       "      <td>[{'args': {'question': 'When is his birthday?'...</td>\n",
       "      <td>[{'args': {'source': [[{'page_content': '# PER...</td>\n",
       "      <td>6</td>\n",
       "      <td>1182</td>\n",
       "      <td>0.03331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ResumeRetrieverApplication</td>\n",
       "      <td>{\"tru_class_info\": {\"name\": \"TruChain\", \"modul...</td>\n",
       "      <td>RunnableSequence(langchain_core.runnables.base)</td>\n",
       "      <td>record_hash_dcf3c0e91420daa1c5f6072e76f1756f</td>\n",
       "      <td>\"When is his mobile number?\"</td>\n",
       "      <td>\"Patrick Guillano La Rosa's mobile number is +...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_dcf3c0e91420daa1c5f...</td>\n",
       "      <td>{\"n_requests\": 3, \"n_successful_requests\": 3, ...</td>\n",
       "      <td>{\"start_time\": \"2024-05-14T14:23:52.436247\", \"...</td>\n",
       "      <td>2024-05-14T14:23:57.745053</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>[{'args': {'prompt': 'When is his mobile numbe...</td>\n",
       "      <td>[{'args': {'question': 'When is his mobile num...</td>\n",
       "      <td>[{'args': {'source': [[{'page_content': '# PER...</td>\n",
       "      <td>5</td>\n",
       "      <td>1160</td>\n",
       "      <td>0.03263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ResumeRetrieverApplication</td>\n",
       "      <td>{\"tru_class_info\": {\"name\": \"TruChain\", \"modul...</td>\n",
       "      <td>RunnableSequence(langchain_core.runnables.base)</td>\n",
       "      <td>record_hash_daf5fde5a65b01f13aed41d0da3455b1</td>\n",
       "      <td>\"When are his hobbies?\"</td>\n",
       "      <td>\"Please contact Patrick La Rosa for details ab...</td>\n",
       "      <td>-</td>\n",
       "      <td>{\"record_id\": \"record_hash_daf5fde5a65b01f13ae...</td>\n",
       "      <td>{\"n_requests\": 3, \"n_successful_requests\": 3, ...</td>\n",
       "      <td>{\"start_time\": \"2024-05-14T14:23:58.019085\", \"...</td>\n",
       "      <td>2024-05-14T14:24:03.137690</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>1170</td>\n",
       "      <td>0.03298</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       app_id  \\\n",
       "0  ResumeRetrieverApplication   \n",
       "1  ResumeRetrieverApplication   \n",
       "2  ResumeRetrieverApplication   \n",
       "3  ResumeRetrieverApplication   \n",
       "4  ResumeRetrieverApplication   \n",
       "5  ResumeRetrieverApplication   \n",
       "6  ResumeRetrieverApplication   \n",
       "7  ResumeRetrieverApplication   \n",
       "8  ResumeRetrieverApplication   \n",
       "9  ResumeRetrieverApplication   \n",
       "\n",
       "                                            app_json  \\\n",
       "0  {\"tru_class_info\": {\"name\": \"TruChain\", \"modul...   \n",
       "1  {\"tru_class_info\": {\"name\": \"TruChain\", \"modul...   \n",
       "2  {\"tru_class_info\": {\"name\": \"TruChain\", \"modul...   \n",
       "3  {\"tru_class_info\": {\"name\": \"TruChain\", \"modul...   \n",
       "4  {\"tru_class_info\": {\"name\": \"TruChain\", \"modul...   \n",
       "5  {\"tru_class_info\": {\"name\": \"TruChain\", \"modul...   \n",
       "6  {\"tru_class_info\": {\"name\": \"TruChain\", \"modul...   \n",
       "7  {\"tru_class_info\": {\"name\": \"TruChain\", \"modul...   \n",
       "8  {\"tru_class_info\": {\"name\": \"TruChain\", \"modul...   \n",
       "9  {\"tru_class_info\": {\"name\": \"TruChain\", \"modul...   \n",
       "\n",
       "                                              type  \\\n",
       "0  RunnableSequence(langchain_core.runnables.base)   \n",
       "1  RunnableSequence(langchain_core.runnables.base)   \n",
       "2  RunnableSequence(langchain_core.runnables.base)   \n",
       "3  RunnableSequence(langchain_core.runnables.base)   \n",
       "4  RunnableSequence(langchain_core.runnables.base)   \n",
       "5  RunnableSequence(langchain_core.runnables.base)   \n",
       "6  RunnableSequence(langchain_core.runnables.base)   \n",
       "7  RunnableSequence(langchain_core.runnables.base)   \n",
       "8  RunnableSequence(langchain_core.runnables.base)   \n",
       "9  RunnableSequence(langchain_core.runnables.base)   \n",
       "\n",
       "                                      record_id  \\\n",
       "0  record_hash_b727a00d3855f32b55678a653a4a192f   \n",
       "1  record_hash_bf8ca5f284718dbf6fc4f0caebc171ea   \n",
       "2  record_hash_32117b90f955318c0a1d982c251850d8   \n",
       "3  record_hash_61030784c623d05423e7b2d1984d7240   \n",
       "4  record_hash_60a8a754d52f9ae639533abd2bf4a5e0   \n",
       "5  record_hash_9c817439e3e1d0a104d18d8b3fbd6ac8   \n",
       "6  record_hash_05c6af17e5e86415f8fd82ab7cbf5251   \n",
       "7  record_hash_7ccf30171368929b740781f2c79e7bb7   \n",
       "8  record_hash_dcf3c0e91420daa1c5f6072e76f1756f   \n",
       "9  record_hash_daf5fde5a65b01f13aed41d0da3455b1   \n",
       "\n",
       "                                           input  \\\n",
       "0           \"Can you share his work experience?\"   \n",
       "1  \"Does he have experience with generative ai?\"   \n",
       "2               \"What is his latest employment?\"   \n",
       "3                       \"What is his first job?\"   \n",
       "4                   \"total years of experience?\"   \n",
       "5            \"What is his professional summary?\"   \n",
       "6  \"What is his highest educational attainment?\"   \n",
       "7                        \"When is his birthday?\"   \n",
       "8                   \"When is his mobile number?\"   \n",
       "9                        \"When are his hobbies?\"   \n",
       "\n",
       "                                              output tags  \\\n",
       "0  \"Certainly! Here's a summary of the work exper...    -   \n",
       "1  \"Yes, he has experience with generative AI. He...    -   \n",
       "2  \"The latest employment is at TDCX, where the i...    -   \n",
       "3  \"The first job mentioned in the resume is the ...    -   \n",
       "4  \"Based on the resume provided, Patrick La Rosa...    -   \n",
       "5  \"Patrick is an engineer by education, a data s...    -   \n",
       "6  \"Patrick La Rosa's highest educational attainm...    -   \n",
       "7  \"The resume does not provide information regar...    -   \n",
       "8  \"Patrick Guillano La Rosa's mobile number is +...    -   \n",
       "9  \"Please contact Patrick La Rosa for details ab...    -   \n",
       "\n",
       "                                         record_json  \\\n",
       "0  {\"record_id\": \"record_hash_b727a00d3855f32b556...   \n",
       "1  {\"record_id\": \"record_hash_bf8ca5f284718dbf6fc...   \n",
       "2  {\"record_id\": \"record_hash_32117b90f955318c0a1...   \n",
       "3  {\"record_id\": \"record_hash_61030784c623d05423e...   \n",
       "4  {\"record_id\": \"record_hash_60a8a754d52f9ae6395...   \n",
       "5  {\"record_id\": \"record_hash_9c817439e3e1d0a104d...   \n",
       "6  {\"record_id\": \"record_hash_05c6af17e5e86415f8f...   \n",
       "7  {\"record_id\": \"record_hash_7ccf30171368929b740...   \n",
       "8  {\"record_id\": \"record_hash_dcf3c0e91420daa1c5f...   \n",
       "9  {\"record_id\": \"record_hash_daf5fde5a65b01f13ae...   \n",
       "\n",
       "                                           cost_json  \\\n",
       "0  {\"n_requests\": 3, \"n_successful_requests\": 3, ...   \n",
       "1  {\"n_requests\": 3, \"n_successful_requests\": 3, ...   \n",
       "2  {\"n_requests\": 3, \"n_successful_requests\": 3, ...   \n",
       "3  {\"n_requests\": 3, \"n_successful_requests\": 3, ...   \n",
       "4  {\"n_requests\": 3, \"n_successful_requests\": 3, ...   \n",
       "5  {\"n_requests\": 3, \"n_successful_requests\": 3, ...   \n",
       "6  {\"n_requests\": 3, \"n_successful_requests\": 3, ...   \n",
       "7  {\"n_requests\": 3, \"n_successful_requests\": 3, ...   \n",
       "8  {\"n_requests\": 3, \"n_successful_requests\": 3, ...   \n",
       "9  {\"n_requests\": 3, \"n_successful_requests\": 3, ...   \n",
       "\n",
       "                                           perf_json  \\\n",
       "0  {\"start_time\": \"2024-05-14T14:22:28.171547\", \"...   \n",
       "1  {\"start_time\": \"2024-05-14T14:23:01.813168\", \"...   \n",
       "2  {\"start_time\": \"2024-05-14T14:23:11.475942\", \"...   \n",
       "3  {\"start_time\": \"2024-05-14T14:23:19.206538\", \"...   \n",
       "4  {\"start_time\": \"2024-05-14T14:23:25.129293\", \"...   \n",
       "5  {\"start_time\": \"2024-05-14T14:23:30.589785\", \"...   \n",
       "6  {\"start_time\": \"2024-05-14T14:23:38.892871\", \"...   \n",
       "7  {\"start_time\": \"2024-05-14T14:23:45.360881\", \"...   \n",
       "8  {\"start_time\": \"2024-05-14T14:23:52.436247\", \"...   \n",
       "9  {\"start_time\": \"2024-05-14T14:23:58.019085\", \"...   \n",
       "\n",
       "                           ts  relevance  context_relevance_with_cot_reasons  \\\n",
       "0  2024-05-14T14:23:01.413212        1.0                                 0.9   \n",
       "1  2024-05-14T14:23:11.056413        0.8                                 0.9   \n",
       "2  2024-05-14T14:23:18.797011        1.0                                 1.0   \n",
       "3  2024-05-14T14:23:24.755373        1.0                                 0.8   \n",
       "4  2024-05-14T14:23:30.275122        1.0                                 0.8   \n",
       "5  2024-05-14T14:23:38.568031        0.9                                 1.0   \n",
       "6  2024-05-14T14:23:44.948560        1.0                                 1.0   \n",
       "7  2024-05-14T14:23:52.098814        1.0                                 0.2   \n",
       "8  2024-05-14T14:23:57.745053        1.0                                 1.0   \n",
       "9  2024-05-14T14:24:03.137690        NaN                                 NaN   \n",
       "\n",
       "   groundedness_measure_with_cot_reasons  \\\n",
       "0                               0.653846   \n",
       "1                               0.000000   \n",
       "2                               0.000000   \n",
       "3                               0.000000   \n",
       "4                               0.000000   \n",
       "5                               1.000000   \n",
       "6                               1.000000   \n",
       "7                               0.500000   \n",
       "8                               1.000000   \n",
       "9                                    NaN   \n",
       "\n",
       "                                     relevance_calls  \\\n",
       "0  [{'args': {'prompt': 'Can you share his work e...   \n",
       "1  [{'args': {'prompt': 'Does he have experience ...   \n",
       "2  [{'args': {'prompt': 'What is his latest emplo...   \n",
       "3  [{'args': {'prompt': 'What is his first job?',...   \n",
       "4  [{'args': {'prompt': 'total years of experienc...   \n",
       "5  [{'args': {'prompt': 'What is his professional...   \n",
       "6  [{'args': {'prompt': 'What is his highest educ...   \n",
       "7  [{'args': {'prompt': 'When is his birthday?', ...   \n",
       "8  [{'args': {'prompt': 'When is his mobile numbe...   \n",
       "9                                                NaN   \n",
       "\n",
       "            context_relevance_with_cot_reasons_calls  \\\n",
       "0  [{'args': {'question': 'Can you share his work...   \n",
       "1  [{'args': {'question': 'Does he have experienc...   \n",
       "2  [{'args': {'question': 'What is his latest emp...   \n",
       "3  [{'args': {'question': 'What is his first job?...   \n",
       "4  [{'args': {'question': 'total years of experie...   \n",
       "5  [{'args': {'question': 'What is his profession...   \n",
       "6  [{'args': {'question': 'What is his highest ed...   \n",
       "7  [{'args': {'question': 'When is his birthday?'...   \n",
       "8  [{'args': {'question': 'When is his mobile num...   \n",
       "9                                                NaN   \n",
       "\n",
       "         groundedness_measure_with_cot_reasons_calls  latency  total_tokens  \\\n",
       "0  [{'args': {'source': [[{'page_content': '- Rec...       33          2167   \n",
       "1  [{'args': {'source': [[{'page_content': '- Rec...        9          1815   \n",
       "2  [{'args': {'source': [[{'page_content': '## Ca...        7          1753   \n",
       "3  [{'args': {'source': [[{'page_content': 'Busin...        5          1813   \n",
       "4  [{'args': {'source': [[{'page_content': 'Busin...        5          1799   \n",
       "5  [{'args': {'source': [[{'page_content': '# PRO...        7          1281   \n",
       "6  [{'args': {'source': [[{'page_content': '# EDU...        6          1198   \n",
       "7  [{'args': {'source': [[{'page_content': '# PER...        6          1182   \n",
       "8  [{'args': {'source': [[{'page_content': '# PER...        5          1160   \n",
       "9                                                NaN        5          1170   \n",
       "\n",
       "   total_cost  \n",
       "0     0.05141  \n",
       "1     0.04087  \n",
       "2     0.03913  \n",
       "3     0.03901  \n",
       "4     0.03886  \n",
       "5     0.03530  \n",
       "6     0.03313  \n",
       "7     0.03331  \n",
       "8     0.03263  \n",
       "9     0.03298  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recs = recording.records \n",
    "\n",
    "records, feedback = tru.get_records_and_feedback(app_ids=[\"ResumeRetrieverApplication\"])\n",
    "\n",
    "records.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a993929c-3578-460d-9fda-1838032c4201",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>relevance</th>\n",
       "      <th>context_relevance_with_cot_reasons</th>\n",
       "      <th>groundedness_measure_with_cot_reasons</th>\n",
       "      <th>latency</th>\n",
       "      <th>total_cost</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>app_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ResumeRetrieverApplication</th>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.844444</td>\n",
       "      <td>0.461538</td>\n",
       "      <td>8.8</td>\n",
       "      <td>0.037663</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            relevance  context_relevance_with_cot_reasons  \\\n",
       "app_id                                                                      \n",
       "ResumeRetrieverApplication   0.966667                            0.844444   \n",
       "\n",
       "                            groundedness_measure_with_cot_reasons  latency  \\\n",
       "app_id                                                                       \n",
       "ResumeRetrieverApplication                               0.461538      8.8   \n",
       "\n",
       "                            total_cost  \n",
       "app_id                                  \n",
       "ResumeRetrieverApplication    0.037663  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tru.get_leaderboard(app_ids=[\"ResumeRetrieverApplication\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}