Skip to content

Commit

Permalink
added updated robustness exp for llama405b
Browse files Browse the repository at this point in the history
  • Loading branch information
@qcampbel
qcampbel committed Jan 24, 2025
1 parent 7d9208a commit 154a8ac
Show file tree
Hide file tree
Showing 10 changed files with 7,998 additions and 702 deletions.
145 changes: 123 additions & 22 deletions notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt1.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -12,12 +12,23 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"('Complete all of the following tasks: 1. Download the PDB file 1LYZ.',\n",
" 'Download the PDB file 1LYZ.')"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import sys\n",
"import os\n",
"parent_dir = os.path.abspath(os.path.join(os.getcwd(), '..'))\n",
"sys.path.append(parent_dir)\n",
"from robustness_prompts import get_prompt\n",
Expand All @@ -30,7 +41,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -40,68 +51,158 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 10,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Thought: The first task is to download the PDB file 1LYZ. I will use the PDBFileDownloader tool to accomplish this.\n",
"\n",
"Action:\n",
"```\n",
"{\n",
" \"action\": \"PDBFileDownloader\",\n",
" \"action_input\": \"1LYZ\"\n",
"}\n",
"```PDB file found with this ID: 1LYZ\n",
"Thought: The PDB file 1LYZ has been downloaded successfully, and I have been given the new file ID \"1LYZ_101021\". \n",
"\n",
"Action:\n",
"```\n",
"{\n",
" \"action\": \"Final Answer\",\n",
" \"action_input\": \"The PDB file 1LYZ has been downloaded successfully with the new file ID: 1LYZ_101021\"\n",
"}\n",
"```"
]
},
{
"data": {
"text/plain": [
"({'input': '\\n You are an expert molecular dynamics scientist, and\\n your task is to respond to the question or\\n solve the problem to the best of your ability using\\n the provided tools.\\n\\n You can only respond with a single complete\\n \\'Thought, Action, Action Input\\' format\\n OR a single \\'Final Answer\\' format.\\n\\n Complete format:\\n Thought: (reflect on your progress and decide what to do next)\\n Action:\\n ```\\n {\\n \"action\": (the action name, it should be the name of a tool),\\n \"action_input\": (the input string for the action)\\n }\\n \\'\\'\\'\\n\\n OR\\n\\n Final Answer: (the final response to the original input\\n question, once all steps are complete)\\n\\n You are required to use the tools provided,\\n using the most specific tool\\n available for each action.\\n Your final answer should contain all information\\n necessary to answer the question and its subquestions.\\n Before you finish, reflect on your progress and make\\n sure you have addressed the question in its entirety.\\n\\n If you are asked to continue\\n or reference previous runs,\\n the context will be provided to you.\\n If context is provided, you should assume\\n you are continuing a chat.\\n\\n Here is the input:\\n Previous Context: None\\n Question: Complete all of the following tasks: 1. Download the PDB file 1LYZ. ',\n",
" 'output': 'The PDB file 1LYZ has been downloaded successfully with the new file ID: 1LYZ_101021'},\n",
" 'SDH872QP')"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"agent_1 = MDAgent(agent_type=\"Structured\", model=llm_model, top_k_tools=tools, use_memory=False)\n",
"agent_1.run(prompt_1_descriptive)"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 11,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Files found in registry: 1LYZ_101021: PDB file downloaded from RSCB, PDBFile ID: 1LYZ_101021\n"
]
}
],
"source": [
"registry_1 = agent_1.path_registry\n",
"print(registry_1.list_path_names_and_descriptions())"
]
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"# assert os.path.exists(registry_1.get_mapped_path(\"<PDB>\"))"
"assert os.path.exists(registry_1.get_mapped_path(\"1LYZ_101021\"))"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 16,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Thought: The problem requires downloading a PDB file with the ID 1LYZ. The PDBFileDownloader tool can be used for this purpose.\n",
"\n",
"Action:\n",
"```\n",
"{\n",
" \"action\": \"PDBFileDownloader\",\n",
" \"action_input\": \"1LYZ\"\n",
"}\n",
"```PDB file found with this ID: 1LYZ\n",
"Thought: The PDB file 1LYZ has been successfully downloaded, and the resulting file ID is 1LYZ_101147. This file ID can be used for further analysis or processing.\n",
"\n",
"Action:\n",
"```\n",
"{\n",
" \"action\": \"Final Answer\",\n",
" \"action_input\": \"The PDB file 1LYZ has been downloaded and saved with the file ID 1LYZ_101147.\"\n",
"}\n",
"```"
]
},
{
"data": {
"text/plain": [
"({'input': '\\n You are an expert molecular dynamics scientist, and\\n your task is to respond to the question or\\n solve the problem to the best of your ability using\\n the provided tools.\\n\\n You can only respond with a single complete\\n \\'Thought, Action, Action Input\\' format\\n OR a single \\'Final Answer\\' format.\\n\\n Complete format:\\n Thought: (reflect on your progress and decide what to do next)\\n Action:\\n ```\\n {\\n \"action\": (the action name, it should be the name of a tool),\\n \"action_input\": (the input string for the action)\\n }\\n \\'\\'\\'\\n\\n OR\\n\\n Final Answer: (the final response to the original input\\n question, once all steps are complete)\\n\\n You are required to use the tools provided,\\n using the most specific tool\\n available for each action.\\n Your final answer should contain all information\\n necessary to answer the question and its subquestions.\\n Before you finish, reflect on your progress and make\\n sure you have addressed the question in its entirety.\\n\\n If you are asked to continue\\n or reference previous runs,\\n the context will be provided to you.\\n If context is provided, you should assume\\n you are continuing a chat.\\n\\n Here is the input:\\n Previous Context: None\\n Question: Download the PDB file 1LYZ. ',\n",
" 'output': 'The PDB file 1LYZ has been downloaded and saved with the file ID 1LYZ_101147.'},\n",
" 'UTOJ735S')"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"agent_2 = MDAgent(agent_type=\"Structured\", model=llm_model, top_k_tools=tools, use_memory=False)\n",
"agent_2.run(prompt_1_natural)"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 18,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Files found in registry: 1LYZ_101147: PDB file downloaded from RSCB, PDBFile ID: 1LYZ_101147\n"
]
}
],
"source": [
"registry_2 = agent_1.path_registry\n",
"registry_2 = agent_2.path_registry\n",
"print(registry_2.list_path_names_and_descriptions())"
]
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"# assert os.path.exists(registry_2.get_mapped_path(\"<PDB>\"))"
"assert os.path.exists(registry_2.get_mapped_path(\"1LYZ_101147\"))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "mdagent2",
"display_name": "mdagent",
"language": "python",
"name": "python3"
"name": "mdagent"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -113,9 +214,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.5"
"version": "3.12.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
"nbformat_minor": 4
}
Loading

0 comments on commit 154a8ac

Please sign in to comment.