Merge branch 'experiments' of github.com:ur-whitelab/md-agent into cl…

…aude-experiments
ur-whitelab · Oct 17, 2024 · 90651f6 · 90651f6
2 parents dfc3351 + 28e114e
commit 90651f6
Show file tree

Hide file tree

Showing 250 changed files with 228,284 additions and 4,182 deletions.
diff --git a/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt1.ipynb b/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt1.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt2.ipynb b/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt2.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt3.ipynb b/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt3.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt4.ipynb b/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt4.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt5.ipynb b/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt5.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt6.ipynb b/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt6.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt7.ipynb b/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt7.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt8.ipynb b/notebooks/experiments/Robustness/llama-v3p1-405b-instruct/prompt8.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt1.ipynb b/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt1.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt2.ipynb b/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt2.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt3.ipynb b/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt3.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt4.ipynb b/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt4.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt5.ipynb b/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt5.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt6.ipynb b/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt6.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt7.ipynb b/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt7.ipynb
diff --git a/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt8.ipynb b/notebooks/experiments/Robustness/llama-v3p1-70b-instruct/prompt8.ipynb
diff --git a/notebooks/experiments/experiment_k1/gpt-3.5-turbo-0125/exp_1.ipynb b/notebooks/experiments/experiment_k1/gpt-3.5-turbo-0125/exp_1.ipynb
@@ -102,7 +102,7 @@
     "#rmsd2ID = 'fig0_165231'\n",
     "path1 = registry.get_mapped_path(rmsd1ID)\n",
     "#path2 = registry.get_mapped_path(rmsd2ID)\n",
-    "\n",
+    "#change the dir to \n",
     "Image(filename=path1)\n",
     "\n",
     "\n"
@@ -155,10 +155,12 @@
    "metadata": {},
    "source": [
     "# Experiment Result:\n",
-    "### Completed without Exception or TimeOut Errors ✅\n",
+    "### Completed without Exception or TimeOut Errors ❌\n",
     "### Attempted all necessary steps ❌\n",
-    "### Logic make sense ❌\n",
-    "### Correct Answer  ❌"
+    "### Logic make sense ✅\n",
+    "### Correct Answer  ❌\n",
+    "### Number of steps completed: 3\n",
+    "### % steps completed: 37.50%"
    ]
   },
   {

diff --git a/notebooks/experiments/experiment_k1/gpt-3.5-turbo-0125/exp_10.ipynb b/notebooks/experiments/experiment_k1/gpt-3.5-turbo-0125/exp_10.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "39451d99-005f-40b2-867e-fb955d63647d",
    "metadata": {},
    "outputs": [],
@@ -12,72 +12,97 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "d808d0af-27f5-4e95-87bf-beb2d72c9108",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "date: 2024-09-10\n",
+      "time: 10:43:18\n"
+     ]
+    }
+   ],
    "source": [
     "#todays date and time\n",
     "import datetime\n",
     "\n",
-    "now = datetime.datetime.now()\n",
-    "date = now.strftime(\"%Y-%m-%d\")\n",
-    "print(\"date and time:\",date)\n",
-    "time = now.strftime(\"%H:%M:%S\")\n",
+    "start = datetime.datetime.now()\n",
+    "date = start.strftime(\"%Y-%m-%d\")\n",
+    "print(\"date:\",date)\n",
+    "time = start.strftime(\"%H:%M:%S\")\n",
     "print(\"time:\",time)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "82be5f35-3197-49c7-9955-6707529f8691",
    "metadata": {},
    "outputs": [],
    "source": [
-    "prompt10 = \"Download the PDB file for 1AEE. Then, tell me how many chains and atoms are in the protein.\"\n",
     "model = \"gpt-3.5-turbo-0125\"\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "4b12fdc0-25be-4835-9e70-b3ea299bac9d",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "LLM:  gpt-3.5-turbo-0125 \n",
+      "Temperature:  0.1\n"
+     ]
+    }
+   ],
    "source": [
     "mda = MDAgent(\n",
     "    agent_type='Structured', \n",
     "    model=model, \n",
     "    tools_model=model,\n",
     "    use_memory=False, \n",
-    "    top_k_tools=\"all\"\n",
+    "    top_k_tools=\"all\",\n",
+    "    ckpt_dir='ckpt_77'\n",
     ")\n",
     "\n",
     "print(\"LLM: \", mda.llm.model_name,\"\\nTemperature: \", mda.llm.temperature)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "fe11721e-c775-40d3-9332-9f80820ccc95",
    "metadata": {},
    "outputs": [],
    "source": [
-    "answer = mda.run(prompt10)"
+    "prompt = 'Simulate protein 1ZNI at 300 K for 1 ns and calculate the RMSD.'\n",
+    "# answer = mda.run(prompt)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "81d4d4a1-091e-49db-8272-33c2a11a984b",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "duration: 0.00 minutes\n"
+     ]
+    }
+   ],
    "source": [
-    "now = datetime.datetime.now()\n",
-    "date = now.strftime(\"%Y-%m-%d\")\n",
-    "print(\"date and time:\",date)\n",
-    "time = now.strftime(\"%H:%M:%S\")\n",
-    "print(\"time:\",time)"
+    "end = datetime.datetime.now()\n",
+    "elapsed = end - start\n",
+    "print(f\"duration: {elapsed.total_seconds()/60:.2f} minutes\")"
    ]
   },
   {
@@ -90,29 +115,61 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "id": "0ba860e5-1dd0-4fcc-aa2f-d5e7f975cbae",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Names found in registry: 1ZNI_173902, 1ZNI_173907, top_sim0_173911, sim0_173911, rec0_173911, rec1_173911, rec2_173911\n"
+     ]
+    }
+   ],
    "source": [
     "registry = mda.path_registry\n",
-    "all_paths = registry._list_all_paths()"
+    "all_names = registry.list_path_names()\n",
+    "print(all_names)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "4b9220de-6a3c-4db9-8768-68298ece63a6",
+   "execution_count": 10,
+   "id": "88df24af-5df3-421c-96a5-895a09161eb0",
    "metadata": {},
    "outputs": [],
    "source": [
+    "# ensure all files are in path registry\n",
+    "# assert all(n in all_names for n in ['1ZNI', 'rmsd', 'sim0', 'top_sim0', 'fig0']), \"Not all file ids are present in path registry\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "21eb1565-59bf-4c0a-a25a-fbd6feda405e",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "AttributeError",
+     "evalue": "'NoneType' object has no attribute 'group'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[12], line 5\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n\u001b[1;32m      4\u001b[0m match \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msearch(\u001b[38;5;124mrf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfig0_\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124md+\u001b[39m\u001b[38;5;124m\"\u001b[39m, all_names)\n\u001b[0;32m----> 5\u001b[0m plot_path \u001b[38;5;241m=\u001b[39m registry\u001b[38;5;241m.\u001b[39mget_mapped_path(\u001b[43mmatch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroup\u001b[49m(\u001b[38;5;241m0\u001b[39m))\n\u001b[1;32m      6\u001b[0m plot_path \u001b[38;5;241m=\u001b[39m registry\u001b[38;5;241m.\u001b[39mckpt_dir \u001b[38;5;241m+\u001b[39m plot_path\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mckpt_76\u001b[39m\u001b[38;5;124m\"\u001b[39m)[\u001b[38;5;241m1\u001b[39m]\n\u001b[1;32m      7\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(plot_path), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno plot found\u001b[39m\u001b[38;5;124m'\u001b[39m\n",
+      "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'group'"
+     ]
+    }
+   ],
+   "source": [
+    "# # let's see if we can't grab the plot\n",
     "import re\n",
     "import os\n",
-    "pattern = re.compile(r\"1AEE.*\\.pdb\")\n",
-    "assert len(all_paths) > 0\n",
-    "assert pattern.search(all_paths[0])\n",
-    "assert os.path.exists(all_paths[0])\n",
-    "print('It is asserted that 1AEE pdb is in path_registry')"
+    "match = re.search(rf\"fig0_\\d+\", all_names)\n",
+    "plot_path = registry.get_mapped_path(match.group(0))\n",
+    "plot_path = registry.ckpt_dir + plot_path.split(\"ckpt_76\")[1]\n",
+    "assert os.path.exists(plot_path), 'no plot found'"
    ]
   },
   {
@@ -122,37 +179,30 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import mdtraj as md\n",
-    "traj = md.load(all_paths[0])\n",
-    "top = traj.topology\n",
-    "number_of_chains = top.n_chains\n",
-    "number_of_atoms = top.n_atoms\n",
-    "print('Number of chains: ', number_of_chains)\n",
-    "print('Number of atoms: ', number_of_atoms)"
+    "# from IPython.display import Image\n",
+    "# plot_path = registry.get_mapped_path('fig0_073719')\n",
+    "# Image(filename=plot_path)"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
-   "id": "25deff6d-6c2d-4b63-91df-1b7e109450dc",
+   "id": "790b786b-95e1-4359-a76b-55c20b4cb261",
    "metadata": {},
    "source": [
     "# Experiment Result:\n",
-    "\n",
-    "1. Completed without Exception or TimeOut Errors ✅\n",
-    "2. Attempted all necessary steps ✅\n",
-    "3. Logic make sense ✅\n",
-    "4. Correct Answer ✅"
+    "### Completed without Exception or TimeOut Errors ✅\n",
+    "### Attempted all necessary steps ❌\n",
+    "### Logic make sense ❌\n",
+    "### No hallucination ✅\n",
+    "### Correct Answer  ❌\n",
+    "### Number of steps completed: 2\n",
+    "### % steps completed: 66.67%\n"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "bff27c19",
-   "metadata": {},
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "id": "9c5dad4b",
+   "id": "7d032e29",
    "metadata": {},
    "source": []
   }