analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "DEBUG = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_funct(base_directory):\n",
    "\n",
    "    \n",
    "    # Iterate over all directories in the base directory\n",
    "    for response_dir in os.listdir(base_directory):\n",
    "        response_path = os.path.join(base_directory, response_dir)\n",
    "        \n",
    "        # Check if it's a directory and matches the response-* pattern\n",
    "        if os.path.isdir(response_path) and response_dir.startswith(\"response-\"):\n",
    "            testing_file = os.path.join(response_path, \"testing.json\")\n",
    "            \n",
    "            # Check if the testing.json file exists\n",
    "            if os.path.exists(testing_file):\n",
    "                with open(testing_file, \"r\") as f:\n",
    "                    try:\n",
    "                        testing_data = json.load(f)\n",
    "                        if not testing_data[\"deployed_successful\"] and testing_data[\"deploy_errors\"]:\n",
    "                            print(f\"Deployment Error: {testing_data['deploy_errors']} in {testing_file}\")\n",
    "                                        \n",
    "\n",
    "                        \n",
    "                    except json.JSONDecodeError:\n",
    "                        raise Exception(f\"Error: Failed to parse JSON in {testing_file}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_k8sgpt(k8sgpt_file):\n",
    "    with open(k8sgpt_file, \"r\") as f:\n",
    "        try:\n",
    "            k8sgpt_data = json.load(f)\n",
    "            if k8sgpt_data[\"results\"]:\n",
    "                for result in k8sgpt_data[\"results\"]:\n",
    "                    if result.get(\"kind\") == \"PersistentVolumeClaim\":\n",
    "                        for error in result.get(\"error\", []):\n",
    "                            print(f\"PVC error: {error['Text']} in {k8sgpt_file}\")\n",
    "                            return \"PVC_failed\"\n",
    "                for result in k8sgpt_data[\"results\"]:\n",
    "                    if result.get(\"kind\") == \"Pod\":\n",
    "                        for error in result.get(\"error\", []):\n",
    "                            if \"Back-off pulling image\" in error[\"Text\"]:\n",
    "                                # raise Exception\n",
    "                                raise Exception(f\"Back-off pulling image in {k8sgpt_file}\")\n",
    "                            elif \"not found\" in error[\"Text\"] and (\"persistentvolumeclaim\" in error[\"Text\"] or \"secret\" in error[\"Text\"] or \"configmap\" in error[\"Text\"]):\n",
    "                                DEBUG and print(f\"Missing resources: {error['Text']} in {k8sgpt_file}\")\n",
    "                                return \"Missing_resources\"\n",
    "                            elif \"couldn't find key\" in error[\"Text\"]:\n",
    "                                DEBUG and print(f\"Missing key in secret or configmap: {error['Text']} in {k8sgpt_file}\")\n",
    "                                return \"Missing_key\"\n",
    "                for result in k8sgpt_data[\"results\"]:\n",
    "                    if result.get(\"kind\") == \"Pod\":\n",
    "                        if \"mysql\" in result.get(\"parentObject\"):\n",
    "                            for error in result.get(\"error\", []):\n",
    "                                if \"PersistentVolumeClaims\" in error[\"Text\"]:\n",
    "                                    DEBUG and print(f\"MySQL pod has unbound immediate PersistentVolumeClaims in {k8sgpt_file}\")\n",
    "                                    return \"MySQL_PVC_unbound\"\n",
    "                                elif \"Readiness probe failed\" in error[\"Text\"]:\n",
    "                                    DEBUG and print(f\"MySQL Readiness probe failed in {k8sgpt_file}\")\n",
    "                                    return \"MySQL_not_ready\"\n",
    "                for result in k8sgpt_data[\"results\"]:\n",
    "                    if result.get(\"kind\") == \"Pod\":\n",
    "                        if \"wordpress\" in result.get(\"parentObject\"):\n",
    "                            for error in result.get(\"error\", []):\n",
    "                                if \"PersistentVolumeClaims\" in error[\"Text\"]:\n",
    "                                    print(f\"Wordpress pod has unbound immediate PersistentVolumeClaims in {k8sgpt_file}\")\n",
    "                                    return \"Wordress_PVC_unbound\"\n",
    "                                elif \"Readiness probe failed\" in error[\"Text\"]:\n",
    "                                    DEBUG and print(f\"MySQL Readiness probe failed in {k8sgpt_file}\")\n",
    "                                    return \"Wordpress_not_ready\"\n",
    "                for result in k8sgpt_data[\"results\"]:\n",
    "                    if result.get(\"kind\") != \"Service\":\n",
    "                        for error in result.get(\"error\", []):\n",
    "                            print(f\"Unknown error in: {error['Text']} in {k8sgpt_file}\")\n",
    "                            return \"Unknown_error\"\n",
    "                for result in k8sgpt_data[\"results\"]:\n",
    "                    if result.get(\"kind\") == \"Service\":\n",
    "                        for error in result.get(\"error\", []):\n",
    "                            print(f\"Service error: {error['Text']} in {k8sgpt_file}\")\n",
    "                            return \"Service_failed\"\n",
    "                print(f\"Unknown error in {k8sgpt_file}\")\n",
    "                return \"Unknown_error\"\n",
    "        except json.JSONDecodeError:\n",
    "            raise Exception(f\"Error: Failed to parse JSON in {k8sgpt_file}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os  \n",
    "import json\n",
    "from collections import defaultdict\n",
    "\n",
    "def aggregate_test_results(base_directory):\n",
    "\n",
    "    aggregated_results = {\n",
    "        \"total_responses\": 0,\n",
    "        \"secrets_used\": 0,\n",
    "        \"base64_encoding_needed\": 0,\n",
    "        \"yaml_not_wrapped\": 0,\n",
    "        \"invalid_yaml\": 0,\n",
    "        \"kubeconform_failed\": 0,\n",
    "        \"no_mysql_or_wordpress\": 0,\n",
    "        \"deployment_failed\": 0,\n",
    "        \"duplicate_resources\": 0,\n",
    "        \"healthy\": 0,\n",
    "        \"unhealthy\": 0,\n",
    "        \"PVC_failed\": 0,\n",
    "        \"Missing_resources\": 0,\n",
    "        \"Missing_key\": 0,\n",
    "        \"MySQL_PVC_unbound\": 0,\n",
    "        \"MySQL_not_ready\": 0,\n",
    "        \"Wordpress_PVC_unbound\": 0,\n",
    "        \"Wordpress_not_ready\": 0,\n",
    "        \"Service_failed\": 0,\n",
    "        \"Unknown_error\": 0\n",
    "    }\n",
    "    scores_healthy = []\n",
    "    \n",
    "    # Iterate over all directories in the base directory\n",
    "    for response_dir in os.listdir(base_directory):\n",
    "        response_path = os.path.join(base_directory, response_dir)\n",
    "        \n",
    "        # Check if it's a directory and matches the response-* pattern\n",
    "        if os.path.isdir(response_path) and response_dir.startswith(\"response-\"):\n",
    "            testing_file = os.path.join(response_path, \"testing.json\")\n",
    "            \n",
    "            # Check if the testing.json file exists\n",
    "            if os.path.exists(testing_file):\n",
    "                with open(testing_file, \"r\") as f:\n",
    "                    try:\n",
    "                        testing_data = json.load(f)\n",
    "\n",
    "                        # Aggregate results  \n",
    "                        aggregated_results[\"total_responses\"] += 1\n",
    "\n",
    "                        # Get extra information\n",
    "                        if testing_data.get(\"secrets_found\"):\n",
    "                            aggregated_results[\"secrets_used\"] += 1\n",
    "                            DEBUG and print(f\"secrets used in {response_path}\")\n",
    "\n",
    "                        if testing_data.get(\"base64_needed\"):\n",
    "                            aggregated_results[\"base64_encoding_needed\"] += 1\n",
    "\n",
    "                        if testing_data.get(\"yaml_not_wrapped\"):\n",
    "                            aggregated_results[\"yaml_not_wrapped\"] += 1\n",
    "                            DEBUG and print(f\"YAML not wrapped in {response_path}\")\n",
    "\n",
    "                        elif not testing_data.get(\"valid_yaml\"):\n",
    "                            aggregated_results[\"invalid_yaml\"] += 1\n",
    "                        \n",
    "                        elif not testing_data.get(\"kubeconform\"):\n",
    "                            aggregated_results[\"kubeconform_failed\"] += 1\n",
    "                            kubeconform_file = os.path.join(response_path, \"conform.json\")\n",
    "                            if not os.path.exists(kubeconform_file):\n",
    "                                raise Exception(f\"conform.json not found in {response_path}\")\n",
    "\n",
    "                        elif not testing_data.get(\"mysql_found\") or not testing_data.get(\"wordpress_found\"):\n",
    "                            aggregated_results[\"no_mysql_or_wordpress\"] += 1\n",
    "\n",
    "                        elif not testing_data.get(\"deployed_successful\"):\n",
    "                            aggregated_results[\"deployment_failed\"] += 1\n",
    "\n",
    "                            for deployment_error in testing_data.get(\"deploy_errors\"):\n",
    "                                DEBUG and print(f\"Deployment failed for {response_path}, error: {deployment_error}\")\n",
    "                                if \"AlreadyExists\" in deployment_error:\n",
    "                                    aggregated_results[\"duplicate_resources\"] += 1\n",
    "                                    break\n",
    "\n",
    "                        elif testing_data.get(\"healthy\"):\n",
    "                            aggregated_results[\"healthy\"] += 1\n",
    "                            scores_healthy.append(int(testing_data.get(\"polaris_score\")))\n",
    "\n",
    "                        else:\n",
    "                            # why did it fail if it was not healthy\n",
    "                            aggregated_results[\"unhealthy\"] += 1\n",
    "\n",
    "                            # Check why it failed in the k8sgpt.json file\n",
    "                            k8sgpt_file = os.path.join(response_path, \"k8sgpt.json\")\n",
    "                            if os.path.exists(k8sgpt_file):\n",
    "                                cause = check_k8sgpt(k8sgpt_file)\n",
    "                                if cause == \"PVC_failed\":\n",
    "                                    aggregated_results[\"PVC_failed\"] += 1\n",
    "                                elif cause == \"Missing_resources\":\n",
    "                                    aggregated_results[\"Missing_resources\"] += 1\n",
    "                                elif cause == \"Missing_key\":\n",
    "                                    aggregated_results[\"Missing_key\"] += 1\n",
    "                                elif cause == \"MySQL_PVC_unbound\":\n",
    "                                    aggregated_results[\"MySQL_PVC_unbound\"] += 1\n",
    "                                elif cause == \"MySQL_not_ready\":\n",
    "                                    aggregated_results[\"MySQL_not_ready\"] += 1\n",
    "                                elif cause == \"Wordpress_PVC_unbound\":\n",
    "                                    aggregated_results[\"Wordpress_PVC_unbound\"] += 1\n",
    "                                elif cause == \"Wordpress_not_ready\":\n",
    "                                    aggregated_results[\"Wordpress_not_ready\"] += 1\n",
    "                                elif cause == \"Service_failed\":\n",
    "                                    aggregated_results[\"Service_failed\"] += 1\n",
    "                                elif cause == \"Unknown_error\":\n",
    "                                    aggregated_results[\"Unknown_error\"] += 1\n",
    "                                else:\n",
    "                                    raise Exception(f\"Something went wrong in {response_path}\")\n",
    "                                \n",
    "                            else:\n",
    "                                raise Exception(f\"k8sgpt.json not found in {response_path}\")\n",
    "                        \n",
    "                    except json.JSONDecodeError:\n",
    "                        raise Exception(f\"Error: Failed to parse JSON in {testing_file}\")\n",
    "            else:\n",
    "                raise Exception(f\"testing.json not found in {response_path}\")\n",
    "    if (aggregated_results[\"unhealthy\"] != aggregated_results[\"PVC_failed\"] + aggregated_results[\"MySQL_PVC_unbound\"] + aggregated_results[\"MySQL_not_ready\"] + aggregated_results[\"Wordpress_PVC_unbound\"] + aggregated_results[\"Wordpress_not_ready\"] + aggregated_results[\"Unknown_error\"] + aggregated_results[\"Service_failed\"] + aggregated_results[\"Missing_resources\"] + aggregated_results[\"Missing_key\"]):\n",
    "        raise Exception(f\"Error count does not match for unhealthy {base_directory}\")\n",
    "    if (aggregated_results[\"total_responses\"] != aggregated_results[\"healthy\"] + aggregated_results[\"unhealthy\"] + aggregated_results[\"deployment_failed\"] + aggregated_results[\"no_mysql_or_wordpress\"] + aggregated_results[\"kubeconform_failed\"] + aggregated_results[\"invalid_yaml\"] + aggregated_results[\"yaml_not_wrapped\"]):\n",
    "        raise Exception(f\"Error count does not match {base_directory}\")\n",
    "    return aggregated_results, scores_healthy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "def aggregate_all_test_results(results):  \n",
    "    # Initialize the aggregated results  \n",
    "    aggregated_results = {\n",
    "        \"total_responses\": 0,\n",
    "        \"secrets_used\": 0,\n",
    "        \"base64_encoding_needed\": 0,\n",
    "        \"yaml_not_wrapped\": 0,\n",
    "        \"invalid_yaml\": 0,\n",
    "        \"kubeconform_failed\": 0,\n",
    "        \"no_mysql_or_wordpress\": 0,\n",
    "        \"deployment_failed\": 0,\n",
    "        \"duplicate_resources\": 0,\n",
    "        \"healthy\": 0,\n",
    "        \"unhealthy\": 0,\n",
    "        \"PVC_failed\": 0,\n",
    "        \"Missing_resources\": 0,\n",
    "        \"Missing_key\": 0,\n",
    "        \"MySQL_PVC_unbound\": 0,\n",
    "        \"MySQL_not_ready\": 0,\n",
    "        \"Wordpress_PVC_unbound\": 0,\n",
    "        \"Wordpress_not_ready\": 0,\n",
    "        \"Service_failed\": 0,\n",
    "        \"Unknown_error\": 0\n",
    "    }\n",
    "    \n",
    "    # Iterate through each result  \n",
    "    for result in results: \n",
    "        for key in aggregated_results:  \n",
    "            if key in result:  \n",
    "                aggregated_results[key] += result[key]\n",
    "    if (aggregated_results[\"unhealthy\"] != aggregated_results[\"PVC_failed\"] + aggregated_results[\"MySQL_PVC_unbound\"] + aggregated_results[\"MySQL_not_ready\"] + aggregated_results[\"Wordpress_PVC_unbound\"] + aggregated_results[\"Wordpress_not_ready\"] + aggregated_results[\"Unknown_error\"] + aggregated_results[\"Service_failed\"] + aggregated_results[\"Missing_resources\"] + aggregated_results[\"Missing_key\"]):\n",
    "        raise Exception(f\"Error count does not match for unhealthy results\")\n",
    "    if (aggregated_results[\"total_responses\"] != aggregated_results[\"healthy\"] + aggregated_results[\"unhealthy\"] + aggregated_results[\"deployment_failed\"] + aggregated_results[\"no_mysql_or_wordpress\"] + aggregated_results[\"kubeconform_failed\"] + aggregated_results[\"invalid_yaml\"] + aggregated_results[\"yaml_not_wrapped\"]):\n",
    "        raise Exception(f\"Error count does not match for all results\")\n",
    "    return aggregated_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Debug test for finding common errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4o/zero_shot/role_system_prompt/response-47/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress\\\\\" is invalid: spec.template.spec.containers[0].envFrom: Invalid value: \\\\\"\\\\\": must specify one of: `configMapRef` or `secretRef`\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueInvalid\",\"message\":\"Invalid value: \\\\\"\\\\\": must specify one of: `configMapRef` or `secretRef`\",\"field\":\"spec.template.spec.containers[0].envFrom\"}]},\"code\":422}\\n'] in ./gpt4o/zero_shot/role_system_prompt_detailed/response-40/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress\\\\\" is invalid: spec.template.spec.containers[0].envFrom: Invalid value: \\\\\"\\\\\": must specify one of: `configMapRef` or `secretRef`\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueInvalid\",\"message\":\"Invalid value: \\\\\"\\\\\": must specify one of: `configMapRef` or `secretRef`\",\"field\":\"spec.template.spec.containers[0].envFrom\"}]},\"code\":422}\\n'] in ./gpt4o/zero_shot/role_best_system_prompt_detailed/response-14/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Pod \\\\\"\\\\\" is invalid: [metadata.name: Required value: name or generateName is required, spec.containers: Required value]\",\"reason\":\"Invalid\",\"details\":{\"kind\":\"Pod\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: name or generateName is required\",\"field\":\"metadata.name\"},{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.containers\"}]},\"code\":422}\\n'] in ./gpt4o/meta/meta_meta_prompt/response-8/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pv-claim\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt4o/tot/tot_prompt_2/response-22/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt4o/tot/tot_prompt_2/response-39/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pv-claim\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt4o/tot/tot_prompt_3/response-15/testing.json\n"
     ]
    }
   ],
   "source": [
    "test_funct(\"./gpt4o/cot/ape_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4o/cot/ape_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4o/cot/human_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4o/cot/human_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4o/zero_shot/baseline_system_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4o/zero_shot/baseline_system_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4o/zero_shot/role_system_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4o/zero_shot/role_system_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4o/zero_shot/role_best_system_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4o/zero_shot/role_best_system_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4o/meta/meta_system_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4o/meta/meta_system_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4o/meta/meta_meta_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4o/meta/meta_meta_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4o/tot/tot_prompt_1\")\n",
    "\n",
    "test_funct(\"./gpt4o/tot/tot_prompt_2\")\n",
    "\n",
    "test_funct(\"./gpt4o/tot/tot_prompt_3\")\n",
    "\n",
    "test_funct(\"./gpt4o/tot/tot_detailed_prompt_1\")\n",
    "\n",
    "test_funct(\"./gpt4o/tot/tot_detailed_prompt_2\")\n",
    "\n",
    "test_funct(\"./gpt4o/tot/tot_detailed_prompt_3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: [spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-storage\\\\\", spec.template.spec.containers[0].volumeMounts[1].name: Not found: \\\\\"config-volume\\\\\"]\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"},{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"config-volume\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[1].name\"}]},\"code\":422}\\n'] in ./gpt4/cot/ape_prompt_detailed/response-19/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Service \\\\\"wordpress-service\\\\\" is invalid: [spec.ports[0].name: Required value, spec.ports[1].name: Required value]\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"Service\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.ports[0].name\"},{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.ports[1].name\"}]},\"code\":422}\\n'] in ./gpt4/zero_shot/baseline_system_prompt_detailed/response-27/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Service \\\\\"wordpress\\\\\" is invalid: [spec.ports[0].name: Required value, spec.ports[1].name: Required value]\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"kind\":\"Service\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.ports[0].name\"},{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.ports[1].name\"}]},\"code\":422}\\n'] in ./gpt4/zero_shot/baseline_system_prompt_detailed/response-2/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt4/zero_shot/role_system_prompt/response-15/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt4/meta/meta_system_prompt/response-31/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-18/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-20/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt4/tot/tot_prompt_2/response-26/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-42/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-46/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-29/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-32/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-38/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pv-claim\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-6/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-11/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-0/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-45/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-49/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-7/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-19/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-23/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-43/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-33/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-2/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-39/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-17/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-pass\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pass\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-10/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-48/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_2/response-8/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-pass\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pass\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_3/response-18/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"wordpress-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-pass\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pass\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"wordpress-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt4/tot/tot_prompt_3/response-38/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_3/response-0/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_3/response-28/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_prompt_3/response-17/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_detailed_prompt_2/response-24/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n'] in ./gpt4/tot/tot_detailed_prompt_2/response-18/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_detailed_prompt_2/response-26/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n'] in ./gpt4/tot/tot_detailed_prompt_2/response-42/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt4/tot/tot_detailed_prompt_2/response-36/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n'] in ./gpt4/tot/tot_detailed_prompt_2/response-21/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n'] in ./gpt4/tot/tot_detailed_prompt_2/response-44/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt4/tot/tot_detailed_prompt_2/response-25/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"mysql-db-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-db-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-credentials\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-credentials\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt4/tot/tot_detailed_prompt_2/response-35/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-pass\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pass\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n'] in ./gpt4/tot/tot_detailed_prompt_3/response-43/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt4/tot/tot_detailed_prompt_3/response-2/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.metadata.labels: Invalid value: map[string]string(nil): `selector` does not match template `labels`\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueInvalid\",\"message\":\"Invalid value: map[string]string(nil): `selector` does not match template `labels`\",\"field\":\"spec.template.metadata.labels\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt4/tot/tot_detailed_prompt_3/response-12/testing.json\n"
     ]
    }
   ],
   "source": [
    "test_funct(\"./gpt4/cot/ape_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4/cot/ape_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4/cot/human_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4/cot/human_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4/zero_shot/baseline_system_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4/zero_shot/baseline_system_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4/zero_shot/role_system_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4/zero_shot/role_system_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4/zero_shot/role_best_system_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4/zero_shot/role_best_system_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4/meta/meta_system_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4/meta/meta_system_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4/meta/meta_meta_prompt\")\n",
    "\n",
    "test_funct(\"./gpt4/meta/meta_meta_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt4/tot/tot_prompt_1\")\n",
    "\n",
    "test_funct(\"./gpt4/tot/tot_prompt_2\")\n",
    "\n",
    "test_funct(\"./gpt4/tot/tot_prompt_3\")\n",
    "\n",
    "test_funct(\"./gpt4/tot/tot_detailed_prompt_1\")\n",
    "\n",
    "test_funct(\"./gpt4/tot/tot_detailed_prompt_2\")\n",
    "\n",
    "test_funct(\"./gpt4/tot/tot_detailed_prompt_3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pvc\\\\\" is invalid: spec.accessModes: Required value: at least 1 access mode is required\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: at least 1 access mode is required\",\"field\":\"spec.accessModes\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"wordpress-pvc\\\\\" is invalid: spec.accessModes: Required value: at least 1 access mode is required\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-pvc\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: at least 1 access mode is required\",\"field\":\"spec.accessModes\"}]},\"code\":422}\\n'] in ./gpt3_5/cot/ape_prompt/response-20/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.metadata.labels: Invalid value: map[string]string(nil): `selector` does not match template `labels`\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueInvalid\",\"message\":\"Invalid value: map[string]string(nil): `selector` does not match template `labels`\",\"field\":\"spec.template.metadata.labels\"}]},\"code\":422}\\n'] in ./gpt3_5/cot/ape_prompt_detailed/response-27/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/cot/ape_prompt_detailed/response-4/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-pvc\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-pvc\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-pvc\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-pvc\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/cot/ape_prompt_detailed/response-31/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/cot/human_prompt/response-49/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-pvc\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-pvc\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/cot/human_prompt_detailed/response-11/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/baseline_system_prompt/response-6/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/baseline_system_prompt/response-7/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/baseline_system_prompt/response-33/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/baseline_system_prompt/response-47/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/baseline_system_prompt/response-17/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"mysql-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"wordpress-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/baseline_system_prompt/response-1/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"mysql-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/baseline_system_prompt_detailed/response-18/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"mysql-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/baseline_system_prompt_detailed/response-26/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"mysql-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"wordpress-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/baseline_system_prompt_detailed/response-9/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n'] in ./gpt3_5/zero_shot/baseline_system_prompt_detailed/response-25/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"mysql-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/baseline_system_prompt_detailed/response-13/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n'] in ./gpt3_5/zero_shot/baseline_system_prompt_detailed/response-8/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt/response-22/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"wordpress-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt/response-21/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt/response-29/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt/response-27/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"wordpress-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt/response-4/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"wordpress-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt/response-9/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt/response-35/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"wordpress-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt/response-3/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt/response-13/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt/response-17/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt/response-10/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"wordpress-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt/response-14/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt/response-12/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt_detailed/response-33/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"mysql-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"wordpress-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_system_prompt_detailed/response-25/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"wordpress-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_best_system_prompt/response-5/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_best_system_prompt/response-19/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_best_system_prompt/response-9/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolumeClaim \\\\\"mysql-pv-claim\\\\\" is invalid: spec.resources[storage]: Required value\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"PersistentVolumeClaim\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.resources[storage]\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_best_system_prompt/response-39/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pv-claim\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n'] in ./gpt3_5/zero_shot/role_best_system_prompt_detailed/response-27/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-pvc\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-pvc\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/zero_shot/role_best_system_prompt_detailed/response-16/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/meta/meta_system_prompt/response-5/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/meta/meta_system_prompt/response-36/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/meta/meta_system_prompt/response-32/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"wordpress-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n'] in ./gpt3_5/meta/meta_system_prompt/response-43/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/meta/meta_system_prompt_detailed/response-5/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"mysql-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/meta/meta_system_prompt_detailed/response-34/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/meta/meta_meta_prompt/response-29/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wp-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wp-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/meta/meta_meta_prompt/response-39/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"pods \\\\\"wordpress-mysql-pod\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql-pod\",\"kind\":\"pods\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"pods \\\\\"wordpress-mysql-pod\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql-pod\",\"kind\":\"pods\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"pods \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"pods\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_1/response-45/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/tot/tot_prompt_1/response-49/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_1/response-7/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"pods \\\\\"wordpress-mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql\",\"kind\":\"pods\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_1/response-44/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"pods \\\\\"wordpress-mysql-pod\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql-pod\",\"kind\":\"pods\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_1/response-10/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_2/response-38/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_2/response-11/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-mysql-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-mysql-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_2/response-43/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"mysql-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"namespaces \\\\\"wordpress-namespace\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-namespace\",\"kind\":\"namespaces\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumes \\\\\"mysql-pv\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"persistentvolumes\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_2/response-34/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-mysql-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_2/response-16/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_2/response-25/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/tot/tot_prompt_2/response-2/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_2/response-39/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"namespaces \\\\\"wordpress-namespace\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-namespace\",\"kind\":\"namespaces\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumes \\\\\"mysql-pv\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"persistentvolumes\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumes \\\\\"wordpress-pv\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-pv\",\"kind\":\"persistentvolumes\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"wordpress-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_2/response-8/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"namespaces \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"namespaces\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"namespaces \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"namespaces\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-20/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-26/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-5/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-mysql-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-46/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-29/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-32/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-mysql-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-23/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Service \\\\\"wordpress-mysql-service\\\\\" is invalid: [spec.ports[0].name: Required value, spec.ports[1].name: Required value]\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-mysql-service\",\"kind\":\"Service\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.ports[0].name\"},{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.ports[1].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-mysql-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-15/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-40/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"namespaces \\\\\"wordpress-namespace\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-namespace\",\"kind\":\"namespaces\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"namespaces \\\\\"wordpress-namespace\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-namespace\",\"kind\":\"namespaces\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-25/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"pods \\\\\"wordpress-mysql-pod\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql-pod\",\"kind\":\"pods\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-30/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-31/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"pods \\\\\"wordpress-mysql-pod\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql-pod\",\"kind\":\"pods\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-13/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-mysql-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-47/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-14/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-mysql-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-1/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_prompt_3/response-12/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_1/response-36/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: [spec.selector: Required value, spec.template.metadata.labels: Invalid value: map[string]string(nil): `selector` does not match template `labels`, spec.template.spec.containers: Required value]\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.selector\"},{\"reason\":\"FieldValueInvalid\",\"message\":\"Invalid value: map[string]string(nil): `selector` does not match template `labels`\",\"field\":\"spec.template.metadata.labels\"},{\"reason\":\"FieldValueRequired\",\"message\":\"Required value\",\"field\":\"spec.template.spec.containers\"}]},\"code\":422}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_1/response-46/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_1/response-45/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"mysql-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"mysql-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_1/response-15/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress-deployment\\\\\" is invalid: spec.template.spec.containers[0].env[1].valueFrom.configMapKeyRef.key: Invalid value: \\\\\"WORD:PRESS_DB_USER\\\\\": a valid config key must consist of alphanumeric characters, \\'-\\', \\'_\\' or \\'.\\' (e.g. \\'key.name\\',  or \\'KEY_NAME\\',  or \\'key-name\\', regex used for validation is \\'[-._a-zA-Z0-9]+\\')\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueInvalid\",\"message\":\"Invalid value: \\\\\"WORD:PRESS_DB_USER\\\\\": a valid config key must consist of alphanumeric characters, \\'-\\', \\'_\\' or \\'.\\' (e.g. \\'key.name\\',  or \\'KEY_NAME\\',  or \\'key-name\\', regex used for validation is \\'[-._a-zA-Z0-9]+\\')\",\"field\":\"spec.template.spec.containers[0].env[1].valueFrom.configMapKeyRef.key\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_1/response-40/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_1/response-16/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secrets\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secrets\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_1/response-10/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Pod \\\\\"mysql\\\\\" is invalid: spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"kind\":\"Pod\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_1/response-14/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"wordpress-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_2/response-20/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"mysql-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_2/response-26/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumes \\\\\"mysql-pv\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"persistentvolumes\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_2/response-22/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"wordpress-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"wordpress-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"wordpress-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_2/response-32/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_2/response-11/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_2/response-23/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_2/response-44/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumes \\\\\"mysql-pv\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"persistentvolumes\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_2/response-34/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"PersistentVolume \\\\\"mysql-pv\\\\\" is invalid: spec: Required value: must specify a volume type\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"PersistentVolume\",\"causes\":[{\"reason\":\"FieldValueRequired\",\"message\":\"Required value: must specify a volume type\",\"field\":\"spec\"}]},\"code\":422}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_2/response-40/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_2/response-41/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secrets\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secrets\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"mysql-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"wordpress-secrets\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-secrets\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"wordpress-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_2/response-17/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pv-claim\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_2/response-14/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_2/response-8/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-service\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_3/response-5/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pv-claim\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"wordpress-secrets\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-secrets\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secrets\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secrets\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pv-claim\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pv-claim\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"mysql\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_3/response-36/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-service\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_3/response-34/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress-deployment\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-deployment\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"Deployment.apps \\\\\"mysql-deployment\\\\\" is invalid: spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"reason\":\"Invalid\",\"details\":{\"name\":\"mysql-deployment\",\"group\":\"apps\",\"kind\":\"Deployment\",\"causes\":[{\"reason\":\"FieldValueNotFound\",\"message\":\"Not found: \\\\\"mysql-persistent-storage\\\\\"\",\"field\":\"spec.template.spec.containers[0].volumeMounts[0].name\"}]},\"code\":422}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"mysql-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-service\",\"kind\":\"services\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_3/response-13/testing.json\n",
      "Deployment Error: ['{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"secrets \\\\\"mysql-secret\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-secret\",\"kind\":\"secrets\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"configmaps \\\\\"wordpress-config\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-config\",\"kind\":\"configmaps\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumes \\\\\"mysql-pv\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pv\",\"kind\":\"persistentvolumes\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"persistentvolumeclaims \\\\\"mysql-pvc\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"mysql-pvc\",\"kind\":\"persistentvolumeclaims\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"services \\\\\"wordpress-service\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress-service\",\"kind\":\"services\"},\"code\":409}\\n', '{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"metadata\":{},\"status\":\"Failure\",\"message\":\"deployments.apps \\\\\"wordpress\\\\\" already exists\",\"reason\":\"AlreadyExists\",\"details\":{\"name\":\"wordpress\",\"group\":\"apps\",\"kind\":\"deployments\"},\"code\":409}\\n'] in ./gpt3_5/tot/tot_detailed_prompt_3/response-17/testing.json\n"
     ]
    }
   ],
   "source": [
    "test_funct(\"./gpt3_5/cot/ape_prompt\")\n",
    "\n",
    "test_funct(\"./gpt3_5/cot/ape_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt3_5/cot/human_prompt\")\n",
    "\n",
    "test_funct(\"./gpt3_5/cot/human_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt3_5/zero_shot/baseline_system_prompt\")\n",
    "\n",
    "test_funct(\"./gpt3_5/zero_shot/baseline_system_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt3_5/zero_shot/role_system_prompt\")\n",
    "\n",
    "test_funct(\"./gpt3_5/zero_shot/role_system_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt3_5/zero_shot/role_best_system_prompt\")\n",
    "\n",
    "test_funct(\"./gpt3_5/zero_shot/role_best_system_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt3_5/meta/meta_system_prompt\")\n",
    "\n",
    "test_funct(\"./gpt3_5/meta/meta_system_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt3_5/meta/meta_meta_prompt\")\n",
    "\n",
    "test_funct(\"./gpt3_5/meta/meta_meta_prompt_detailed\")\n",
    "\n",
    "test_funct(\"./gpt3_5/tot/tot_prompt_1\")\n",
    "\n",
    "test_funct(\"./gpt3_5/tot/tot_prompt_2\")\n",
    "\n",
    "test_funct(\"./gpt3_5/tot/tot_prompt_3\")\n",
    "\n",
    "test_funct(\"./gpt3_5/tot/tot_detailed_prompt_1\")\n",
    "\n",
    "test_funct(\"./gpt3_5/tot/tot_detailed_prompt_2\")\n",
    "\n",
    "test_funct(\"./gpt3_5/tot/tot_detailed_prompt_3\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Aggregated test results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT-4o: Zero-Shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt4o/zero_shot/baseline_system_prompt/response-8/k8sgpt.json\n",
      "Aggregated Results: ./gpt4o/zero_shot/baseline_system_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 31,\n",
      "    \"base64_encoding_needed\": 3,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 25,\n",
      "    \"unhealthy\": 25,\n",
      "    \"PVC_failed\": 1,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 24,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [40, 40, 37, 37, 35, 36, 40, 40, 40, 40, 37, 35, 37, 40, 35, 35, 40, 37, 35, 35, 43, 35, 40, 40, 40]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/zero_shot/baseline_system_prompt\"\n",
    "gpt4o_zs_baseline_system_prompt_results, gpt4o_zs_baseline_system_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")  \n",
    "print(json.dumps(gpt4o_zs_baseline_system_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_zs_baseline_system_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4o/zero_shot/baseline_system_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 49,\n",
      "    \"base64_encoding_needed\": 1,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 43,\n",
      "    \"unhealthy\": 7,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 7,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [42, 42, 42, 44, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 44, 42, 44, 44, 42, 42, 39, 44, 42, 42, 42, 42, 42, 39, 44, 42, 44, 42, 42, 42, 42, 42, 44, 44, 39, 42, 42, 42, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/zero_shot/baseline_system_prompt_detailed\"\n",
    "gpt4o_zs_baseline_system_prompt_detailed_results, gpt4o_zs_baseline_system_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")  \n",
    "print(json.dumps(gpt4o_zs_baseline_system_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_zs_baseline_system_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4o/zero_shot/role_system_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 35,\n",
      "    \"base64_encoding_needed\": 1,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 1,\n",
      "    \"duplicate_resources\": 1,\n",
      "    \"healthy\": 24,\n",
      "    \"unhealthy\": 25,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 25,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [40, 35, 35, 40, 40, 40, 40, 41, 40, 35, 35, 35, 40, 38, 40, 40, 35, 40, 37, 40, 40, 36, 40, 38]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/zero_shot/role_system_prompt\"\n",
    "gpt4o_zs_role_system_prompt_results, gpt4o_zs_role_system_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_zs_role_system_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_zs_role_system_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4o/zero_shot/role_system_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 48,\n",
      "    \"base64_encoding_needed\": 5,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 1,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 43,\n",
      "    \"unhealthy\": 6,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 1,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 5,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [42, 44, 40, 42, 42, 39, 42, 42, 44, 44, 42, 42, 42, 42, 42, 44, 42, 42, 42, 42, 42, 42, 42, 42, 42, 44, 42, 44, 42, 42, 42, 44, 44, 42, 42, 42, 44, 42, 44, 42, 42, 42, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/zero_shot/role_system_prompt_detailed\"\n",
    "gpt4o_zs_role_system_prompt_detailed_results, gpt4o_zs_role_system_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_zs_role_system_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_zs_role_system_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4o/zero_shot/role_best_system_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 36,\n",
      "    \"base64_encoding_needed\": 2,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 30,\n",
      "    \"unhealthy\": 19,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 19,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [36, 40, 37, 35, 40, 40, 40, 40, 35, 37, 35, 40, 40, 40, 41, 40, 40, 35, 41, 40, 40, 40, 37, 43, 40, 35, 37, 40, 40, 40]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/zero_shot/role_best_system_prompt\"\n",
    "gpt4o_zs_role_best_system_prompt_results, gpt4o_zs_role_best_system_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_zs_role_best_system_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_zs_role_best_system_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4o/zero_shot/role_best_system_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 45,\n",
      "    \"base64_encoding_needed\": 10,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 1,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 40,\n",
      "    \"unhealthy\": 9,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 9,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [44, 44, 42, 42, 42, 42, 42, 39, 42, 42, 42, 42, 42, 42, 42, 44, 42, 44, 44, 44, 42, 42, 44, 42, 42, 44, 42, 42, 44, 44, 44, 42, 42, 44, 44, 44, 42, 42, 44, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/zero_shot/role_best_system_prompt_detailed\"\n",
    "gpt4o_zs_role_best_system_prompt_detailed_results, gpt4o_zs_role_best_system_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_zs_role_best_system_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_zs_role_best_system_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT-3.5: Zero-Shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt3_5/zero_shot/baseline_system_prompt/response-44/k8sgpt.json\n",
      "Aggregated Results: ./gpt3_5/zero_shot/baseline_system_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 1,\n",
      "    \"base64_encoding_needed\": 0,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 3,\n",
      "    \"deployment_failed\": 6,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 3,\n",
      "    \"unhealthy\": 37,\n",
      "    \"PVC_failed\": 1,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 36,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [35, 35, 32]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/zero_shot/baseline_system_prompt\"\n",
    "gpt3_5_zs_baseline_system_prompt_results, gpt3_5_zs_baseline_system_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_zs_baseline_system_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_zs_baseline_system_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt3_5/zero_shot/baseline_system_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 48,\n",
      "    \"base64_encoding_needed\": 24,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 1,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 6,\n",
      "    \"duplicate_resources\": 2,\n",
      "    \"healthy\": 0,\n",
      "    \"unhealthy\": 43,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 12,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 31,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: []\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/zero_shot/baseline_system_prompt_detailed\"\n",
    "gpt3_5_zs_baseline_system_prompt_detailed_results, gpt3_5_zs_baseline_system_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_zs_baseline_system_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_zs_baseline_system_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt3_5/zero_shot/role_system_prompt/response-18/k8sgpt.json\n",
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt3_5/zero_shot/role_system_prompt/response-5/k8sgpt.json\n",
      "PVC error: storageclass.storage.k8s.io \"my-local-storage\" not found in ./gpt3_5/zero_shot/role_system_prompt/response-19/k8sgpt.json\n",
      "Aggregated Results: ./gpt3_5/zero_shot/role_system_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 0,\n",
      "    \"base64_encoding_needed\": 0,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 13,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 1,\n",
      "    \"unhealthy\": 36,\n",
      "    \"PVC_failed\": 3,\n",
      "    \"Missing_resources\": 1,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 32,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [35]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/zero_shot/role_system_prompt\"\n",
    "gpt3_5_zs_role_system_prompt_results, gpt3_5_zs_role_system_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_zs_role_system_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_zs_role_system_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt3_5/zero_shot/role_system_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 44,\n",
      "    \"base64_encoding_needed\": 20,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 2,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 3,\n",
      "    \"unhealthy\": 45,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 13,\n",
      "    \"Missing_key\": 1,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 31,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [39, 38, 38]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/zero_shot/role_system_prompt_detailed\"\n",
    "gpt3_5_zs_role_system_prompt_detailed_results, gpt3_5_zs_role_system_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_zs_role_system_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_zs_role_system_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt3_5/zero_shot/role_best_system_prompt/response-21/k8sgpt.json\n",
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt3_5/zero_shot/role_best_system_prompt/response-40/k8sgpt.json\n",
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt3_5/zero_shot/role_best_system_prompt/response-25/k8sgpt.json\n",
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt3_5/zero_shot/role_best_system_prompt/response-47/k8sgpt.json\n",
      "Aggregated Results: ./gpt3_5/zero_shot/role_best_system_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 0,\n",
      "    \"base64_encoding_needed\": 0,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 1,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 4,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 0,\n",
      "    \"unhealthy\": 45,\n",
      "    \"PVC_failed\": 4,\n",
      "    \"Missing_resources\": 1,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 40,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: []\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/zero_shot/role_best_system_prompt\"\n",
    "gpt3_5_zs_role_best_system_prompt_results, gpt3_5_zs_role_best_system_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_zs_role_best_system_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_zs_role_best_system_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt3_5/zero_shot/role_best_system_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 41,\n",
      "    \"base64_encoding_needed\": 21,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 2,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 2,\n",
      "    \"duplicate_resources\": 1,\n",
      "    \"healthy\": 1,\n",
      "    \"unhealthy\": 45,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 8,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 37,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [41]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/zero_shot/role_best_system_prompt_detailed\"\n",
    "gpt3_5_zs_role_best_system_prompt_detailed_results, gpt3_5_zs_role_best_system_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_zs_role_best_system_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_zs_role_best_system_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT-4: Zero-Shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/zero_shot/baseline_system_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 24,\n",
      "    \"base64_encoding_needed\": 11,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 10,\n",
      "    \"unhealthy\": 39,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 4,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 35,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [40, 35, 35, 35, 35, 35, 35, 35, 35, 35]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/zero_shot/baseline_system_prompt\"\n",
    "gpt4_zs_baseline_system_prompt_results, gpt4_zs_baseline_system_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_zs_baseline_system_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_zs_baseline_system_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/zero_shot/baseline_system_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 45,\n",
      "    \"base64_encoding_needed\": 12,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 4,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 2,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 29,\n",
      "    \"unhealthy\": 15,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 1,\n",
      "    \"Missing_key\": 1,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 13,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [39, 42, 42, 44, 39, 42, 42, 44, 43, 42, 42, 42, 44, 42, 39, 42, 42, 42, 42, 43, 42, 42, 44, 42, 42, 43, 42, 42, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/zero_shot/baseline_system_prompt_detailed\"\n",
    "gpt4_zs_baseline_system_prompt_detailed_results, gpt4_zs_baseline_system_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_zs_baseline_system_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_zs_baseline_system_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/zero_shot/role_system_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 27,\n",
      "    \"base64_encoding_needed\": 7,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 1,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 1,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 6,\n",
      "    \"unhealthy\": 42,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 3,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 39,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [35, 40, 35, 37, 35, 40]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/zero_shot/role_system_prompt\"\n",
    "gpt4_zs_role_system_prompt_results, gpt4_zs_role_system_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_zs_role_system_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_zs_role_system_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/zero_shot/role_system_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 50,\n",
      "    \"base64_encoding_needed\": 4,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 35,\n",
      "    \"unhealthy\": 15,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 1,\n",
      "    \"Missing_key\": 1,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 13,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [42, 42, 44, 39, 41, 42, 44, 42, 42, 42, 39, 43, 42, 39, 42, 39, 42, 42, 42, 43, 42, 42, 46, 43, 43, 42, 42, 43, 42, 42, 43, 42, 43, 39, 39]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/zero_shot/role_system_prompt_detailed\"\n",
    "gpt4_zs_role_system_prompt_detailed_results, gpt4_zs_role_system_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_zs_role_system_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_zs_role_system_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/zero_shot/role_best_system_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 42,\n",
      "    \"base64_encoding_needed\": 11,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 5,\n",
      "    \"unhealthy\": 44,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 2,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 42,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [37, 40, 37, 35, 40]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/zero_shot/role_best_system_prompt\"\n",
    "gpt4_zs_role_best_system_prompt_results, gpt4_zs_role_best_system_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_zs_role_best_system_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_zs_role_best_system_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/zero_shot/role_best_system_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 49,\n",
      "    \"base64_encoding_needed\": 7,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 36,\n",
      "    \"unhealthy\": 13,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 2,\n",
      "    \"Missing_key\": 1,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 10,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [41, 44, 42, 42, 42, 41, 46, 42, 44, 42, 42, 42, 42, 42, 42, 42, 44, 42, 42, 42, 39, 42, 42, 42, 42, 42, 42, 42, 42, 42, 44, 42, 42, 42, 42, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/zero_shot/role_best_system_prompt_detailed\"\n",
    "gpt4_zs_role_best_system_prompt_detailed_results, gpt4_zs_role_best_system_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_zs_role_best_system_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_zs_role_best_system_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT-4o: CoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Service error: Service has no endpoints, expected label app=mysql in ./gpt4o/cot/human_prompt/response-29/k8sgpt.json\n",
      "Service error: Service has no endpoints, expected label app=mysql in ./gpt4o/cot/human_prompt/response-13/k8sgpt.json\n",
      "Aggregated Results: ./gpt4o/cot/human_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 35,\n",
      "    \"base64_encoding_needed\": 18,\n",
      "    \"yaml_not_wrapped\": 1,\n",
      "    \"invalid_yaml\": 2,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 34,\n",
      "    \"unhealthy\": 13,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 11,\n",
      "    \"Service_failed\": 2,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [40, 35, 40, 35, 35, 40, 39, 40, 43, 35, 40, 40, 35, 40, 40, 40, 40, 43, 40, 35, 40, 40, 40, 40, 35, 40, 43, 40, 40, 35, 40, 40, 40, 40]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/cot/human_prompt\"\n",
    "gpt4o_cot_human_prompt_results, gpt4o_cot_human_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_cot_human_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_cot_human_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt4o/cot/human_prompt_detailed/response-38/k8sgpt.json\n",
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt4o/cot/human_prompt_detailed/response-44/k8sgpt.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt4o/cot/human_prompt_detailed/response-1/k8sgpt.json\n",
      "Aggregated Results: ./gpt4o/cot/human_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 48,\n",
      "    \"base64_encoding_needed\": 20,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 37,\n",
      "    \"unhealthy\": 12,\n",
      "    \"PVC_failed\": 3,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 9,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [42, 42, 43, 42, 42, 42, 44, 42, 42, 42, 42, 42, 42, 40, 42, 42, 42, 42, 42, 42, 40, 42, 44, 44, 42, 42, 42, 44, 42, 44, 42, 42, 42, 42, 42, 40, 40]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/cot/human_prompt_detailed\"\n",
    "gpt4o_cot_human_prompt_detailed_results, gpt4o_cot_human_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_cot_human_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_cot_human_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4o/cot/ape_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 38,\n",
      "    \"base64_encoding_needed\": 14,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 1,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 30,\n",
      "    \"unhealthy\": 19,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 19,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [40, 35, 40, 40, 40, 43, 40, 40, 40, 35, 40, 40, 35, 40, 40, 40, 40, 40, 40, 37, 40, 40, 40, 35, 40, 35, 43, 40, 40, 40]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/cot/ape_prompt\"\n",
    "gpt4o_cot_ape_prompt_results, gpt4o_cot_ape_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_cot_ape_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_cot_ape_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unknown error in: Liveness probe failed: dial tcp 10.244.1.2:80: connect: connection refused in ./gpt4o/cot/ape_prompt_detailed/response-37/k8sgpt.json\n",
      "Aggregated Results: ./gpt4o/cot/ape_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 47,\n",
      "    \"base64_encoding_needed\": 25,\n",
      "    \"yaml_not_wrapped\": 1,\n",
      "    \"invalid_yaml\": 1,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 34,\n",
      "    \"unhealthy\": 13,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 1,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 11,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 1\n",
      "}\n",
      "Polaris Scores: [42, 42, 42, 42, 44, 42, 44, 40, 39, 42, 42, 42, 42, 42, 40, 42, 42, 42, 42, 42, 42, 44, 42, 42, 42, 44, 42, 44, 42, 40, 44, 44, 42, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/cot/ape_prompt_detailed\"\n",
    "gpt4o_cot_ape_prompt_detailed_results, gpt4o_cot_ape_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_cot_ape_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_cot_ape_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT-4: CoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/cot/human_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 34,\n",
      "    \"base64_encoding_needed\": 23,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 2,\n",
      "    \"no_mysql_or_wordpress\": 2,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 15,\n",
      "    \"unhealthy\": 31,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 31,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [40, 40, 35, 40, 35, 40, 40, 35, 42, 35, 35, 35, 35, 35, 35]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/cot/human_prompt\"\n",
    "gpt4_cot_human_prompt_results, gpt4_cot_human_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_cot_human_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_cot_human_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/cot/human_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 48,\n",
      "    \"base64_encoding_needed\": 23,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 2,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 31,\n",
      "    \"unhealthy\": 17,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 17,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [42, 42, 42, 42, 42, 42, 42, 43, 42, 42, 42, 42, 42, 39, 42, 42, 42, 42, 43, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/cot/human_prompt_detailed\"\n",
    "gpt4_cot_human_prompt_detailed_results, gpt4_cot_human_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_cot_human_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_cot_human_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/cot/ape_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 34,\n",
      "    \"base64_encoding_needed\": 25,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 2,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 1,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 18,\n",
      "    \"unhealthy\": 28,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 2,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 26,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [35, 35, 35, 35, 35, 40, 40, 40, 35, 35, 40, 40, 40, 40, 35, 40, 40, 35]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/cot/ape_prompt\"\n",
    "gpt4_cot_ape_prompt_results, gpt4_cot_ape_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_cot_ape_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_cot_ape_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/cot/ape_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 47,\n",
      "    \"base64_encoding_needed\": 25,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 1,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 26,\n",
      "    \"unhealthy\": 22,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 1,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 21,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [42, 42, 44, 42, 42, 43, 42, 42, 36, 42, 42, 42, 39, 42, 39, 42, 42, 42, 42, 42, 39, 42, 42, 42, 42, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/cot/ape_prompt_detailed\"\n",
    "gpt4_cot_ape_prompt_detailed_results, gpt4_cot_ape_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_cot_ape_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_cot_ape_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT-3.5: CoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt3_5/cot/human_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 10,\n",
      "    \"base64_encoding_needed\": 8,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 1,\n",
      "    \"deployment_failed\": 1,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 9,\n",
      "    \"unhealthy\": 38,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 1,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 37,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [35, 32, 35, 32, 36, 32, 35, 35, 36]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/cot/human_prompt\"\n",
    "gpt3_5_cot_human_prompt_results, gpt3_5_cot_human_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_cot_human_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_cot_human_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt3_5/cot/human_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 41,\n",
      "    \"base64_encoding_needed\": 22,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 4,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 1,\n",
      "    \"deployment_failed\": 1,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 2,\n",
      "    \"unhealthy\": 41,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 2,\n",
      "    \"Missing_key\": 2,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 37,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [36, 39]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/cot/human_prompt_detailed\"\n",
    "gpt3_5_cot_human_prompt_detailed_results, gpt3_5_cot_human_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_cot_human_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_cot_human_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt3_5/cot/ape_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 10,\n",
      "    \"base64_encoding_needed\": 9,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 1,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 1,\n",
      "    \"deployment_failed\": 1,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 10,\n",
      "    \"unhealthy\": 37,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 37,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [32, 32, 32, 32, 32, 32, 32, 32, 35, 32]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/cot/ape_prompt\"\n",
    "gpt3_5_cot_ape_prompt_results, gpt3_5_cot_ape_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_cot_ape_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_cot_ape_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt3_5/cot/ape_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 47,\n",
      "    \"base64_encoding_needed\": 24,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 3,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 7,\n",
      "    \"unhealthy\": 39,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 4,\n",
      "    \"Missing_key\": 2,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 33,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [36, 36, 36, 36, 36, 36, 39]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/cot/ape_prompt_detailed\"\n",
    "gpt3_5_cot_ape_prompt_detailed_results, gpt3_5_cot_ape_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_cot_ape_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_cot_ape_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT-4o: ToT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt4o/tot/tot_prompt_1/response-45/k8sgpt.json\n",
      "Unknown error in: the last termination reason is Error container=wordpress pod=wordpress-677bb76474-959rr in ./gpt4o/tot/tot_prompt_1/response-4/k8sgpt.json\n",
      "Aggregated Results: ./gpt4o/tot/tot_prompt_1\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 29,\n",
      "    \"base64_encoding_needed\": 1,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 1,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 30,\n",
      "    \"unhealthy\": 19,\n",
      "    \"PVC_failed\": 1,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 17,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 1\n",
      "}\n",
      "Polaris Scores: [40, 43, 40, 35, 43, 35, 37, 35, 35, 37, 37, 35, 40, 40, 35, 35, 43, 37, 37, 40, 35, 40, 40, 40, 40, 35, 40, 40, 37, 40]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/tot/tot_prompt_1\"\n",
    "gpt4o_tot_prompt_1_results, gpt4o_tot_prompt_1_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_tot_prompt_1_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_tot_prompt_1_scores}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unknown error in: the last termination reason is Error container=wordpress pod=wordpress-55fbb9b877-lmcg9 in ./gpt4o/tot/tot_prompt_2/response-28/k8sgpt.json\n",
      "Service error: Service has not ready endpoints, pods: [Pod/wordpress-bfdf8c4cb-hd9cv], expected 1 in ./gpt4o/tot/tot_prompt_2/response-8/k8sgpt.json\n",
      "Aggregated Results: ./gpt4o/tot/tot_prompt_2\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 28,\n",
      "    \"base64_encoding_needed\": 0,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 3,\n",
      "    \"deployment_failed\": 2,\n",
      "    \"duplicate_resources\": 2,\n",
      "    \"healthy\": 21,\n",
      "    \"unhealthy\": 23,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 21,\n",
      "    \"Service_failed\": 1,\n",
      "    \"Unknown_error\": 1\n",
      "}\n",
      "Polaris Scores: [40, 40, 40, 35, 40, 35, 35, 40, 37, 35, 35, 35, 43, 40, 35, 35, 37, 43, 40, 35, 40]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/tot/tot_prompt_2\"\n",
    "gpt4o_tot_prompt_2_results, gpt4o_tot_prompt_2_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_tot_prompt_2_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_tot_prompt_2_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt4o/tot/tot_prompt_3/response-43/k8sgpt.json\n",
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt4o/tot/tot_prompt_3/response-47/k8sgpt.json\n",
      "Aggregated Results: ./gpt4o/tot/tot_prompt_3\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 20,\n",
      "    \"base64_encoding_needed\": 1,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 3,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 1,\n",
      "    \"deployment_failed\": 1,\n",
      "    \"duplicate_resources\": 1,\n",
      "    \"healthy\": 26,\n",
      "    \"unhealthy\": 18,\n",
      "    \"PVC_failed\": 2,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 16,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [40, 35, 37, 37, 37, 43, 40, 40, 40, 40, 40, 37, 35, 40, 35, 37, 37, 37, 40, 40, 37, 37, 35, 40, 40, 37]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/tot/tot_prompt_3\"\n",
    "gpt4o_tot_prompt_3_results, gpt4o_tot_prompt_3_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_tot_prompt_3_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_tot_prompt_3_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt4o/tot/tot_detailed_prompt_1/response-27/k8sgpt.json\n",
      "Aggregated Results: ./gpt4o/tot/tot_detailed_prompt_1\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 47,\n",
      "    \"base64_encoding_needed\": 8,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 36,\n",
      "    \"unhealthy\": 14,\n",
      "    \"PVC_failed\": 1,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 13,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [42, 42, 44, 44, 44, 44, 42, 42, 44, 42, 42, 42, 42, 42, 42, 44, 44, 42, 44, 42, 42, 42, 42, 42, 42, 43, 42, 44, 42, 42, 44, 44, 42, 44, 39, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/tot/tot_detailed_prompt_1\"\n",
    "gpt4o_tot_detailed_prompt_1_results, gpt4o_tot_detailed_prompt_1_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_tot_detailed_prompt_1_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_tot_detailed_prompt_1_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unknown error in: Liveness probe failed: dial tcp 10.244.1.2:80: connect: connection refused in ./gpt4o/tot/tot_detailed_prompt_2/response-39/k8sgpt.json\n",
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt4o/tot/tot_detailed_prompt_2/response-12/k8sgpt.json\n",
      "Aggregated Results: ./gpt4o/tot/tot_detailed_prompt_2\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 50,\n",
      "    \"base64_encoding_needed\": 6,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 38,\n",
      "    \"unhealthy\": 12,\n",
      "    \"PVC_failed\": 1,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 10,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 1\n",
      "}\n",
      "Polaris Scores: [42, 42, 42, 42, 44, 42, 42, 42, 42, 42, 42, 42, 40, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 44, 42, 44, 42, 42, 42, 42, 39, 42, 42, 42, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/tot/tot_detailed_prompt_2\"\n",
    "gpt4o_tot_detailed_prompt_2_results, gpt4o_tot_detailed_prompt_2_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_tot_detailed_prompt_2_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_tot_detailed_prompt_2_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4o/tot/tot_detailed_prompt_3\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 45,\n",
      "    \"base64_encoding_needed\": 6,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 1,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 30,\n",
      "    \"unhealthy\": 18,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 1,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 17,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 39, 42, 42, 42, 42, 42, 42, 44, 42, 42, 42, 42, 42, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/tot/tot_detailed_prompt_3\"\n",
    "gpt4o_tot_detailed_prompt_3_results, gpt4o_tot_detailed_prompt_3_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_tot_detailed_prompt_3_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_tot_detailed_prompt_3_scores}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT-4: ToT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/tot/tot_prompt_1\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 9,\n",
      "    \"base64_encoding_needed\": 3,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 2,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 4,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 13,\n",
      "    \"unhealthy\": 31,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 7,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 24,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [37, 35, 37, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/tot/tot_prompt_1\"\n",
    "gpt4_tot_prompt_1_results, gpt4_tot_prompt_1_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_tot_prompt_1_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_tot_prompt_1_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/tot/tot_prompt_2\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 20,\n",
      "    \"base64_encoding_needed\": 6,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 2,\n",
      "    \"kubeconform_failed\": 9,\n",
      "    \"no_mysql_or_wordpress\": 2,\n",
      "    \"deployment_failed\": 24,\n",
      "    \"duplicate_resources\": 23,\n",
      "    \"healthy\": 3,\n",
      "    \"unhealthy\": 10,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 6,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 4,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [35, 40, 35]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/tot/tot_prompt_2\"\n",
    "gpt4_tot_prompt_2_results, gpt4_tot_prompt_2_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_tot_prompt_2_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_tot_prompt_2_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/tot/tot_prompt_3\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 12,\n",
      "    \"base64_encoding_needed\": 7,\n",
      "    \"yaml_not_wrapped\": 2,\n",
      "    \"invalid_yaml\": 5,\n",
      "    \"kubeconform_failed\": 5,\n",
      "    \"no_mysql_or_wordpress\": 9,\n",
      "    \"deployment_failed\": 5,\n",
      "    \"duplicate_resources\": 5,\n",
      "    \"healthy\": 1,\n",
      "    \"unhealthy\": 23,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 8,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 15,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [35]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/tot/tot_prompt_3\"\n",
    "gpt4_tot_prompt_3_results, gpt4_tot_prompt_3_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_tot_prompt_3_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_tot_prompt_3_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/tot/tot_detailed_prompt_1\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 32,\n",
      "    \"base64_encoding_needed\": 14,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 3,\n",
      "    \"no_mysql_or_wordpress\": 14,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 18,\n",
      "    \"unhealthy\": 15,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 1,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 14,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [42, 44, 42, 42, 42, 44, 42, 42, 42, 39, 42, 44, 42, 42, 42, 39, 43, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/tot/tot_detailed_prompt_1\"\n",
    "gpt4_tot_detailed_prompt_1_results, gpt4_tot_detailed_prompt_1_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_tot_detailed_prompt_1_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_tot_detailed_prompt_1_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unknown error in: 0/3 nodes are available: 1 node(s) had untolerated taint {node-role.kubernetes.io/control-plane: }, 2 node(s) had untolerated taint {node.kubernetes.io/not-ready: }. preemption: 0/3 nodes are available: 3 Preemption is not helpful for scheduling. in ./gpt4/tot/tot_detailed_prompt_2/response-22/k8sgpt.json\n",
      "Aggregated Results: ./gpt4/tot/tot_detailed_prompt_2\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 30,\n",
      "    \"base64_encoding_needed\": 8,\n",
      "    \"yaml_not_wrapped\": 1,\n",
      "    \"invalid_yaml\": 5,\n",
      "    \"kubeconform_failed\": 5,\n",
      "    \"no_mysql_or_wordpress\": 8,\n",
      "    \"deployment_failed\": 9,\n",
      "    \"duplicate_resources\": 9,\n",
      "    \"healthy\": 8,\n",
      "    \"unhealthy\": 14,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 5,\n",
      "    \"Missing_key\": 1,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 7,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 1\n",
      "}\n",
      "Polaris Scores: [44, 42, 42, 42, 42, 40, 44, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/tot/tot_detailed_prompt_2\"\n",
    "gpt4_tot_detailed_prompt_2_results, gpt4_tot_detailed_prompt_2_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_tot_detailed_prompt_2_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_tot_detailed_prompt_2_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unknown error in: Liveness probe failed: dial tcp 10.244.1.2:80: connect: connection refused in ./gpt4/tot/tot_detailed_prompt_3/response-40/k8sgpt.json\n",
      "Aggregated Results: ./gpt4/tot/tot_detailed_prompt_3\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 20,\n",
      "    \"base64_encoding_needed\": 9,\n",
      "    \"yaml_not_wrapped\": 7,\n",
      "    \"invalid_yaml\": 5,\n",
      "    \"kubeconform_failed\": 3,\n",
      "    \"no_mysql_or_wordpress\": 15,\n",
      "    \"deployment_failed\": 3,\n",
      "    \"duplicate_resources\": 2,\n",
      "    \"healthy\": 1,\n",
      "    \"unhealthy\": 16,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 6,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 9,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 1\n",
      "}\n",
      "Polaris Scores: [44]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/tot/tot_detailed_prompt_3\"\n",
    "gpt4_tot_detailed_prompt_3_results, gpt4_tot_detailed_prompt_3_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_tot_detailed_prompt_3_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_tot_detailed_prompt_3_scores}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT-3.5: ToT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unknown error in: the last termination reason is Error container=mysql pod=mysql in ./gpt3_5/tot/tot_prompt_1/response-28/k8sgpt.json\n",
      "Unknown error in: the last termination reason is Error container=mysql pod=mysql in ./gpt3_5/tot/tot_prompt_1/response-40/k8sgpt.json\n",
      "Unknown error in: the last termination reason is Error container=mysql pod=mysql in ./gpt3_5/tot/tot_prompt_1/response-12/k8sgpt.json\n",
      "Aggregated Results: ./gpt3_5/tot/tot_prompt_1\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 1,\n",
      "    \"base64_encoding_needed\": 1,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 2,\n",
      "    \"kubeconform_failed\": 5,\n",
      "    \"no_mysql_or_wordpress\": 20,\n",
      "    \"deployment_failed\": 5,\n",
      "    \"duplicate_resources\": 4,\n",
      "    \"healthy\": 0,\n",
      "    \"unhealthy\": 18,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 1,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 14,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 3\n",
      "}\n",
      "Polaris Scores: []\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/tot/tot_prompt_1\"\n",
    "gpt3_5_tot_prompt_1_results, gpt3_5_tot_prompt_1_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_tot_prompt_1_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_tot_prompt_1_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unknown error in: the last termination reason is Error container=mysql pod=wordpress-deployment-66d874d745-cjd4v in ./gpt3_5/tot/tot_prompt_2/response-20/k8sgpt.json\n",
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt3_5/tot/tot_prompt_2/response-23/k8sgpt.json\n",
      "Unknown error in: Ingress default/wordpress-ingress does not specify an Ingress class. in ./gpt3_5/tot/tot_prompt_2/response-35/k8sgpt.json\n",
      "Aggregated Results: ./gpt3_5/tot/tot_prompt_2\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 1,\n",
      "    \"base64_encoding_needed\": 1,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 1,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 21,\n",
      "    \"deployment_failed\": 9,\n",
      "    \"duplicate_resources\": 8,\n",
      "    \"healthy\": 0,\n",
      "    \"unhealthy\": 18,\n",
      "    \"PVC_failed\": 1,\n",
      "    \"Missing_resources\": 1,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 14,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 2\n",
      "}\n",
      "Polaris Scores: []\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/tot/tot_prompt_2\"\n",
    "gpt3_5_tot_prompt_2_results, gpt3_5_tot_prompt_2_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_tot_prompt_2_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_tot_prompt_2_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unknown error in: the last termination reason is Error container=mysql pod=wordpress-mysql-deployment-859994f47d-c67rj in ./gpt3_5/tot/tot_prompt_3/response-0/k8sgpt.json\n",
      "Unknown error in: the last termination reason is Error container=mysql pod=wordpress-mysql-deployment-7c868f8-h7qzr in ./gpt3_5/tot/tot_prompt_3/response-49/k8sgpt.json\n",
      "Aggregated Results: ./gpt3_5/tot/tot_prompt_3\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 1,\n",
      "    \"base64_encoding_needed\": 1,\n",
      "    \"yaml_not_wrapped\": 1,\n",
      "    \"invalid_yaml\": 3,\n",
      "    \"kubeconform_failed\": 2,\n",
      "    \"no_mysql_or_wordpress\": 14,\n",
      "    \"deployment_failed\": 17,\n",
      "    \"duplicate_resources\": 16,\n",
      "    \"healthy\": 0,\n",
      "    \"unhealthy\": 13,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 1,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 10,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 2\n",
      "}\n",
      "Polaris Scores: []\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/tot/tot_prompt_3\"\n",
    "gpt3_5_tot_prompt_3_results, gpt3_5_tot_prompt_3_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_tot_prompt_3_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_tot_prompt_3_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt3_5/tot/tot_detailed_prompt_1\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 39,\n",
      "    \"base64_encoding_needed\": 22,\n",
      "    \"yaml_not_wrapped\": 1,\n",
      "    \"invalid_yaml\": 1,\n",
      "    \"kubeconform_failed\": 2,\n",
      "    \"no_mysql_or_wordpress\": 21,\n",
      "    \"deployment_failed\": 8,\n",
      "    \"duplicate_resources\": 5,\n",
      "    \"healthy\": 0,\n",
      "    \"unhealthy\": 17,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 7,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 10,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: []\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/tot/tot_detailed_prompt_1\"\n",
    "gpt3_5_tot_detailed_prompt_1_results, gpt3_5_tot_detailed_prompt_1_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_tot_detailed_prompt_1_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_tot_detailed_prompt_1_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt3_5/tot/tot_detailed_prompt_2\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 42,\n",
      "    \"base64_encoding_needed\": 23,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 2,\n",
      "    \"kubeconform_failed\": 6,\n",
      "    \"no_mysql_or_wordpress\": 2,\n",
      "    \"deployment_failed\": 13,\n",
      "    \"duplicate_resources\": 10,\n",
      "    \"healthy\": 1,\n",
      "    \"unhealthy\": 26,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 9,\n",
      "    \"Missing_key\": 2,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 15,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [36]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/tot/tot_detailed_prompt_2\"\n",
    "gpt3_5_tot_detailed_prompt_2_results, gpt3_5_tot_detailed_prompt_2_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_tot_detailed_prompt_2_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_tot_detailed_prompt_2_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt3_5/tot/tot_detailed_prompt_3\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 19,\n",
      "    \"base64_encoding_needed\": 13,\n",
      "    \"yaml_not_wrapped\": 2,\n",
      "    \"invalid_yaml\": 3,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 11,\n",
      "    \"deployment_failed\": 5,\n",
      "    \"duplicate_resources\": 5,\n",
      "    \"healthy\": 0,\n",
      "    \"unhealthy\": 28,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 21,\n",
      "    \"Missing_key\": 2,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 5,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: []\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/tot/tot_detailed_prompt_3\"\n",
    "gpt3_5_tot_detailed_prompt_3_results, gpt3_5_tot_detailed_prompt_3_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_tot_detailed_prompt_3_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_tot_detailed_prompt_3_scores}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT-4o: Meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4o/meta/meta_system_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 40,\n",
      "    \"base64_encoding_needed\": 0,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 28,\n",
      "    \"unhealthy\": 22,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 22,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [40, 37, 40, 40, 40, 40, 43, 40, 40, 43, 35, 40, 40, 40, 43, 35, 37, 35, 43, 40, 40, 43, 43, 40, 35, 40, 40, 40]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/meta/meta_system_prompt\"\n",
    "gpt4o_meta_system_prompt_results, gpt4o_meta_system_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_meta_system_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_meta_system_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4o/meta/meta_system_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 49,\n",
      "    \"base64_encoding_needed\": 3,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 44,\n",
      "    \"unhealthy\": 5,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 2,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 3,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [44, 44, 42, 42, 42, 42, 42, 42, 42, 42, 44, 42, 42, 42, 44, 44, 42, 42, 42, 42, 42, 42, 44, 44, 42, 42, 44, 39, 42, 39, 44, 42, 42, 42, 42, 42, 42, 44, 42, 42, 42, 42, 42, 44]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/meta/meta_system_prompt_detailed\"\n",
    "gpt4o_meta_system_prompt_detailed_results, gpt4o_meta_system_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_meta_system_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_meta_system_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unknown error in: Ingress default/wordpress-ingress does not specify an Ingress class. in ./gpt4o/meta/meta_meta_prompt/response-4/k8sgpt.json\n",
      "Unknown error in: the last termination reason is Error container=wordpress pod=wordpress-55f9b5b9d-whtgl in ./gpt4o/meta/meta_meta_prompt/response-3/k8sgpt.json\n",
      "PVC error: storageclass.storage.k8s.io \"fast\" not found in ./gpt4o/meta/meta_meta_prompt/response-12/k8sgpt.json\n",
      "Aggregated Results: ./gpt4o/meta/meta_meta_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 22,\n",
      "    \"base64_encoding_needed\": 7,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 2,\n",
      "    \"kubeconform_failed\": 24,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 1,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 4,\n",
      "    \"unhealthy\": 19,\n",
      "    \"PVC_failed\": 1,\n",
      "    \"Missing_resources\": 2,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 1,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 13,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 2\n",
      "}\n",
      "Polaris Scores: [55, 62, 58, 57]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/meta/meta_meta_prompt\"\n",
    "gpt4o_meta_meta_prompt_results, gpt4o_meta_meta_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_meta_meta_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_meta_meta_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unknown error in: Ingress default/wordpress-ingress does not specify an Ingress class. in ./gpt4o/meta/meta_meta_prompt_detailed/response-24/k8sgpt.json\n",
      "PVC error: storageclass.storage.k8s.io \"gp2\" not found in ./gpt4o/meta/meta_meta_prompt_detailed/response-21/k8sgpt.json\n",
      "Unknown error in: 0/3 nodes are available: 1 node(s) had untolerated taint {node-role.kubernetes.io/control-plane: }, 2 node(s) had untolerated taint {node.kubernetes.io/not-ready: }. preemption: 0/3 nodes are available: 3 Preemption is not helpful for scheduling. in ./gpt4o/meta/meta_meta_prompt_detailed/response-34/k8sgpt.json\n",
      "Unknown error in: Liveness probe failed: dial tcp 10.244.1.4:3306: connect: connection refused in ./gpt4o/meta/meta_meta_prompt_detailed/response-15/k8sgpt.json\n",
      "PVC error: storageclass.storage.k8s.io \"high-performance\" not found in ./gpt4o/meta/meta_meta_prompt_detailed/response-2/k8sgpt.json\n",
      "Unknown error in: Ingress wordpress-app/wordpress-ingress does not specify an Ingress class. in ./gpt4o/meta/meta_meta_prompt_detailed/response-35/k8sgpt.json\n",
      "Unknown error in: 0/3 nodes are available: 1 node(s) had untolerated taint {node-role.kubernetes.io/control-plane: }, 2 node(s) had untolerated taint {node.kubernetes.io/not-ready: }. preemption: 0/3 nodes are available: 3 Preemption is not helpful for scheduling. in ./gpt4o/meta/meta_meta_prompt_detailed/response-31/k8sgpt.json\n",
      "Unknown error in: StatefulSet uses the service default/mysql-headless which does not exist. in ./gpt4o/meta/meta_meta_prompt_detailed/response-13/k8sgpt.json\n",
      "Aggregated Results: ./gpt4o/meta/meta_meta_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 37,\n",
      "    \"base64_encoding_needed\": 14,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 11,\n",
      "    \"no_mysql_or_wordpress\": 6,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 14,\n",
      "    \"unhealthy\": 19,\n",
      "    \"PVC_failed\": 2,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 2,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 9,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 6\n",
      "}\n",
      "Polaris Scores: [59, 60, 44, 51, 46, 59, 45, 46, 54, 46, 51, 61, 56, 41]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4o/meta/meta_meta_prompt_detailed\"\n",
    "gpt4o_meta_meta_prompt_detailed_results, gpt4o_meta_meta_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4o_meta_meta_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4o_meta_meta_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT-4: Meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/meta/meta_system_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 43,\n",
      "    \"base64_encoding_needed\": 9,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 2,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 1,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 8,\n",
      "    \"unhealthy\": 39,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 1,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 38,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [40, 37, 40, 35, 40, 43, 40, 40]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/meta/meta_system_prompt\"\n",
    "gpt4_meta_system_prompt_results, gpt4_meta_system_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_meta_system_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_meta_system_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/meta/meta_system_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 50,\n",
      "    \"base64_encoding_needed\": 3,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 0,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 35,\n",
      "    \"unhealthy\": 15,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 15,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [42, 42, 43, 39, 39, 39, 39, 42, 42, 38, 42, 39, 43, 42, 42, 42, 42, 42, 39, 39, 38, 39, 42, 42, 42, 42, 39, 39, 43, 42, 42, 42, 39, 39, 42]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/meta/meta_system_prompt_detailed\"\n",
    "gpt4_meta_system_prompt_detailed_results, gpt4_meta_system_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_meta_system_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_meta_system_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt4/meta/meta_meta_prompt/response-34/k8sgpt.json\n",
      "Aggregated Results: ./gpt4/meta/meta_meta_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 27,\n",
      "    \"base64_encoding_needed\": 20,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 9,\n",
      "    \"no_mysql_or_wordpress\": 3,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 1,\n",
      "    \"unhealthy\": 37,\n",
      "    \"PVC_failed\": 1,\n",
      "    \"Missing_resources\": 12,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 5,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 19,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [43]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/meta/meta_meta_prompt\"\n",
    "gpt4_meta_meta_prompt_results, gpt4_meta_meta_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_meta_meta_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_meta_meta_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PVC error: storageclass.storage.k8s.io \"high-durability\" not found in ./gpt4/meta/meta_meta_prompt_detailed/response-32/k8sgpt.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt4/meta/meta_meta_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 45,\n",
      "    \"base64_encoding_needed\": 21,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 4,\n",
      "    \"no_mysql_or_wordpress\": 3,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 8,\n",
      "    \"unhealthy\": 35,\n",
      "    \"PVC_failed\": 1,\n",
      "    \"Missing_resources\": 4,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 4,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 26,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [44, 46, 46, 58, 58, 46, 64, 48]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt4/meta/meta_meta_prompt_detailed\"\n",
    "gpt4_meta_meta_prompt_detailed_results, gpt4_meta_meta_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt4_meta_meta_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt4_meta_meta_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT-3.5: Meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt3_5/meta/meta_system_prompt/response-26/k8sgpt.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PVC error: storageclass.storage.k8s.io \"manual\" not found in ./gpt3_5/meta/meta_system_prompt/response-17/k8sgpt.json\n",
      "Aggregated Results: ./gpt3_5/meta/meta_system_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 3,\n",
      "    \"base64_encoding_needed\": 1,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 0,\n",
      "    \"deployment_failed\": 4,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 3,\n",
      "    \"unhealthy\": 42,\n",
      "    \"PVC_failed\": 2,\n",
      "    \"Missing_resources\": 0,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 40,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [36, 36, 36]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/meta/meta_system_prompt\"\n",
    "gpt3_5_meta_system_prompt_results, gpt3_5_meta_system_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_meta_system_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_meta_system_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: ./gpt3_5/meta/meta_system_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 45,\n",
      "    \"base64_encoding_needed\": 26,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 1,\n",
      "    \"no_mysql_or_wordpress\": 1,\n",
      "    \"deployment_failed\": 2,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 1,\n",
      "    \"unhealthy\": 45,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 2,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 1,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 42,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 0\n",
      "}\n",
      "Polaris Scores: [36]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/meta/meta_system_prompt_detailed\"\n",
    "gpt3_5_meta_system_prompt_detailed_results, gpt3_5_meta_system_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_meta_system_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_meta_system_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unknown error in: the last termination reason is Error container=mysql pod=mysql-6787599ff5-lnv2q in ./gpt3_5/meta/meta_meta_prompt/response-32/k8sgpt.json\n",
      "Aggregated Results: ./gpt3_5/meta/meta_meta_prompt\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 4,\n",
      "    \"base64_encoding_needed\": 4,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 0,\n",
      "    \"kubeconform_failed\": 8,\n",
      "    \"no_mysql_or_wordpress\": 14,\n",
      "    \"deployment_failed\": 2,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 2,\n",
      "    \"unhealthy\": 24,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 3,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 2,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 18,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 1\n",
      "}\n",
      "Polaris Scores: [38, 47]\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/meta/meta_meta_prompt\"\n",
    "gpt3_5_meta_meta_prompt_results, gpt3_5_meta_meta_prompt_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_meta_meta_prompt_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_meta_meta_prompt_scores}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unknown error in: the last termination reason is Error container=mysql pod=mysql-statefulset-0 in ./gpt3_5/meta/meta_meta_prompt_detailed/response-6/k8sgpt.json\n",
      "Unknown error in: the last termination reason is Error container=mysql pod=mysql-0 in ./gpt3_5/meta/meta_meta_prompt_detailed/response-17/k8sgpt.json\n",
      "Unknown error in: Ingress default/wordpress-ingress does not specify an Ingress class. in ./gpt3_5/meta/meta_meta_prompt_detailed/response-1/k8sgpt.json\n",
      "Aggregated Results: ./gpt3_5/meta/meta_meta_prompt_detailed\n",
      "{\n",
      "    \"total_responses\": 50,\n",
      "    \"secrets_used\": 4,\n",
      "    \"base64_encoding_needed\": 2,\n",
      "    \"yaml_not_wrapped\": 0,\n",
      "    \"invalid_yaml\": 1,\n",
      "    \"kubeconform_failed\": 28,\n",
      "    \"no_mysql_or_wordpress\": 10,\n",
      "    \"deployment_failed\": 0,\n",
      "    \"duplicate_resources\": 0,\n",
      "    \"healthy\": 0,\n",
      "    \"unhealthy\": 11,\n",
      "    \"PVC_failed\": 0,\n",
      "    \"Missing_resources\": 1,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 0,\n",
      "    \"MySQL_not_ready\": 0,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 7,\n",
      "    \"Service_failed\": 0,\n",
      "    \"Unknown_error\": 3\n",
      "}\n",
      "Polaris Scores: []\n"
     ]
    }
   ],
   "source": [
    "base_directory = \"./gpt3_5/meta/meta_meta_prompt_detailed\"\n",
    "gpt3_5_meta_meta_prompt_detailed_results, gpt3_5_meta_meta_prompt_detailed_scores = aggregate_test_results(base_directory)\n",
    "\n",
    "print(f\"Aggregated Results: {base_directory}\")\n",
    "print(json.dumps(gpt3_5_meta_meta_prompt_detailed_results, indent=4))\n",
    "print(f\"Polaris Scores: {gpt3_5_meta_meta_prompt_detailed_scores}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Code for succes rate + average score table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_average_score(scores):\n",
    "    if len(scores) == 0:\n",
    "        print(f\"Avg score: 0\")\n",
    "    else:\n",
    "        print(f\"Avg score: {sum(scores) / len(scores)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_success_rate(input):\n",
    "    if isinstance(input, list):\n",
    "        results = aggregate_all_test_results(input)\n",
    "    else:\n",
    "        results = input\n",
    "    rate = results[\"healthy\"] / results[\"total_responses\"]\n",
    "    print(f\"Succes rate: {rate}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Code for performance table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT-4o: zero-shot baseline average\n",
      "Succes rate: 0.68\n",
      "Avg score: 40.64705882352941\n",
      "GPT-4o: zero-shot baseline system prompt\n",
      "Succes rate: 0.5\n",
      "Avg score: 37.96\n",
      "GPT-4o: zero-shot baseline system prompt detailed\n",
      "Succes rate: 0.86\n",
      "Avg score: 42.2093023255814\n",
      "GPT-4o: zero-shot role system average\n",
      "Succes rate: 0.67\n",
      "Avg score: 40.91044776119403\n",
      "GPT-4o: zero-shot role system prompt\n",
      "Succes rate: 0.48\n",
      "Avg score: 38.333333333333336\n",
      "GPT-4o: zero-shot role system prompt detailed\n",
      "Succes rate: 0.86\n",
      "Avg score: 42.348837209302324\n",
      "GPT-4o: zero-shot role best system average\n",
      "Succes rate: 0.7\n",
      "Avg score: 41.01428571428571\n",
      "GPT-4o: zero-shot role best system prompt\n",
      "Succes rate: 0.6\n",
      "Avg score: 38.8\n",
      "GPT-4o: zero-shot role best system prompt detailed\n",
      "Succes rate: 0.8\n",
      "Avg score: 42.675\n",
      "GPT-4o: cot human average\n",
      "Succes rate: 0.71\n",
      "Avg score: 40.63380281690141\n",
      "GPT-4o: cot human prompt\n",
      "Succes rate: 0.68\n",
      "Avg score: 39.05882352941177\n",
      "GPT-4o: cot human prompt detailed\n",
      "Succes rate: 0.74\n",
      "Avg score: 42.08108108108108\n",
      "GPT-4o: cot ape average\n",
      "Succes rate: 0.64\n",
      "Avg score: 40.796875\n",
      "GPT-4o: cot ape prompt\n",
      "Succes rate: 0.6\n",
      "Avg score: 39.266666666666666\n",
      "GPT-4o: cot ape prompt detailed\n",
      "Succes rate: 0.68\n",
      "Avg score: 42.14705882352941\n",
      "GPT-4o: Meta system average\n",
      "Succes rate: 0.72\n",
      "Avg score: 41.333333333333336\n",
      "GPT-4o: Meta system prompt\n",
      "Succes rate: 0.56\n",
      "Avg score: 39.714285714285715\n",
      "GPT-4o: Meta system prompt detailed\n",
      "Succes rate: 0.88\n",
      "Avg score: 42.36363636363637\n",
      "GPT-4o: Meta meta average\n",
      "Succes rate: 0.18\n",
      "Avg score: 52.833333333333336\n",
      "GPT-4o: Meta meta prompt\n",
      "Succes rate: 0.08\n",
      "Avg score: 58.0\n",
      "GPT-4o: Meta meta prompt detailed\n",
      "Succes rate: 0.28\n",
      "Avg score: 51.357142857142854\n",
      "GPT-4o: tot prompt 1 average\n",
      "Succes rate: 0.66\n",
      "Avg score: 40.60606060606061\n",
      "GPT-4o: tot prompt 1\n",
      "Succes rate: 0.6\n",
      "Avg score: 38.2\n",
      "GPT-4o: tot detailed prompt 1\n",
      "Succes rate: 0.72\n",
      "Avg score: 42.611111111111114\n",
      "GPT-4o: tot prompt 2 average\n",
      "Succes rate: 0.59\n",
      "Avg score: 40.54237288135593\n",
      "GPT-4o: tot prompt 2\n",
      "Succes rate: 0.42\n",
      "Avg score: 37.857142857142854\n",
      "GPT-4o: tot detailed prompt 2\n",
      "Succes rate: 0.76\n",
      "Avg score: 42.026315789473685\n",
      "GPT-4o: tot prompt 3 average\n",
      "Succes rate: 0.56\n",
      "Avg score: 40.214285714285715\n",
      "GPT-4o: tot prompt 3\n",
      "Succes rate: 0.52\n",
      "Avg score: 38.19230769230769\n",
      "GPT-4o: tot detailed prompt 3\n",
      "Succes rate: 0.6\n",
      "Avg score: 41.96666666666667\n",
      "GPT-4o detailed average\n",
      "Succes rate: 0.718\n",
      "Avg score: 42.63509749303621\n",
      "GPT-4o not detailed average\n",
      "Succes rate: 0.504\n",
      "Avg score: 38.95634920634921\n",
      "GPT-4o zero-shot average\n",
      "Succes rate: 0.648\n",
      "Avg score: 40.858536585365854\n",
      "GPT-4o cot average\n",
      "Succes rate: 0.675\n",
      "Avg score: 40.71111111111111\n",
      "GPT-4o Meta average\n",
      "Succes rate: 0.45\n",
      "Avg score: 43.63333333333333\n",
      "GPT-4o tot average\n",
      "Succes rate: 0.6033333333333334\n",
      "Avg score: 40.46408839779006\n",
      "GPT-4o Totale average\n",
      "Succes rate: 0.5978947368421053\n",
      "Avg score: 41.11783960720131\n"
     ]
    }
   ],
   "source": [
    "# GPT-4o\n",
    "print(\"GPT-4o: zero-shot baseline average\")\n",
    "calculate_success_rate([gpt4o_zs_baseline_system_prompt_results, gpt4o_zs_baseline_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt4o_zs_baseline_system_prompt_scores + gpt4o_zs_baseline_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4o: zero-shot baseline system prompt\")\n",
    "calculate_success_rate(gpt4o_zs_baseline_system_prompt_results)\n",
    "calculate_average_score(gpt4o_zs_baseline_system_prompt_scores)\n",
    "\n",
    "print(\"GPT-4o: zero-shot baseline system prompt detailed\")\n",
    "calculate_success_rate(gpt4o_zs_baseline_system_prompt_detailed_results)\n",
    "calculate_average_score(gpt4o_zs_baseline_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4o: zero-shot role system average\")\n",
    "calculate_success_rate([gpt4o_zs_role_system_prompt_results, gpt4o_zs_role_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt4o_zs_role_system_prompt_scores + gpt4o_zs_role_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4o: zero-shot role system prompt\")\n",
    "calculate_success_rate(gpt4o_zs_role_system_prompt_results)\n",
    "calculate_average_score(gpt4o_zs_role_system_prompt_scores)\n",
    "\n",
    "print(\"GPT-4o: zero-shot role system prompt detailed\")\n",
    "calculate_success_rate(gpt4o_zs_role_system_prompt_detailed_results)\n",
    "calculate_average_score(gpt4o_zs_role_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4o: zero-shot role best system average\")\n",
    "calculate_success_rate([gpt4o_zs_role_best_system_prompt_results, gpt4o_zs_role_best_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt4o_zs_role_best_system_prompt_scores + gpt4o_zs_role_best_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4o: zero-shot role best system prompt\")\n",
    "calculate_success_rate(gpt4o_zs_role_best_system_prompt_results)\n",
    "calculate_average_score(gpt4o_zs_role_best_system_prompt_scores)\n",
    "\n",
    "print(\"GPT-4o: zero-shot role best system prompt detailed\")\n",
    "calculate_success_rate(gpt4o_zs_role_best_system_prompt_detailed_results)\n",
    "calculate_average_score(gpt4o_zs_role_best_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4o: cot human average\")\n",
    "calculate_success_rate([gpt4o_cot_human_prompt_results, gpt4o_cot_human_prompt_detailed_results])\n",
    "calculate_average_score(gpt4o_cot_human_prompt_scores + gpt4o_cot_human_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4o: cot human prompt\")\n",
    "calculate_success_rate(gpt4o_cot_human_prompt_results)\n",
    "calculate_average_score(gpt4o_cot_human_prompt_scores)\n",
    "\n",
    "print(\"GPT-4o: cot human prompt detailed\")\n",
    "calculate_success_rate(gpt4o_cot_human_prompt_detailed_results)\n",
    "calculate_average_score(gpt4o_cot_human_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4o: cot ape average\")\n",
    "calculate_success_rate([gpt4o_cot_ape_prompt_results, gpt4o_cot_ape_prompt_detailed_results])\n",
    "calculate_average_score(gpt4o_cot_ape_prompt_scores + gpt4o_cot_ape_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4o: cot ape prompt\")\n",
    "calculate_success_rate(gpt4o_cot_ape_prompt_results)\n",
    "calculate_average_score(gpt4o_cot_ape_prompt_scores)\n",
    "\n",
    "print(\"GPT-4o: cot ape prompt detailed\")\n",
    "calculate_success_rate(gpt4o_cot_ape_prompt_detailed_results)\n",
    "calculate_average_score(gpt4o_cot_ape_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4o: Meta system average\")\n",
    "calculate_success_rate([gpt4o_meta_system_prompt_results, gpt4o_meta_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt4o_meta_system_prompt_scores + gpt4o_meta_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4o: Meta system prompt\")\n",
    "calculate_success_rate(gpt4o_meta_system_prompt_results)\n",
    "calculate_average_score(gpt4o_meta_system_prompt_scores)\n",
    "\n",
    "print(\"GPT-4o: Meta system prompt detailed\")\n",
    "calculate_success_rate(gpt4o_meta_system_prompt_detailed_results)\n",
    "calculate_average_score(gpt4o_meta_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4o: Meta meta average\")\n",
    "calculate_success_rate([gpt4o_meta_meta_prompt_results, gpt4o_meta_meta_prompt_detailed_results])\n",
    "calculate_average_score(gpt4o_meta_meta_prompt_scores + gpt4o_meta_meta_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4o: Meta meta prompt\")\n",
    "calculate_success_rate(gpt4o_meta_meta_prompt_results)\n",
    "calculate_average_score(gpt4o_meta_meta_prompt_scores)\n",
    "\n",
    "print(\"GPT-4o: Meta meta prompt detailed\")\n",
    "calculate_success_rate(gpt4o_meta_meta_prompt_detailed_results)\n",
    "calculate_average_score(gpt4o_meta_meta_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4o: tot prompt 1 average\")\n",
    "calculate_success_rate([gpt4o_tot_prompt_1_results, gpt4o_tot_detailed_prompt_1_results])\n",
    "calculate_average_score(gpt4o_tot_prompt_1_scores + gpt4o_tot_detailed_prompt_1_scores)\n",
    "\n",
    "print(\"GPT-4o: tot prompt 1\")\n",
    "calculate_success_rate(gpt4o_tot_prompt_1_results)\n",
    "calculate_average_score(gpt4o_tot_prompt_1_scores)\n",
    "\n",
    "print(\"GPT-4o: tot detailed prompt 1\")\n",
    "calculate_success_rate(gpt4o_tot_detailed_prompt_1_results)\n",
    "calculate_average_score(gpt4o_tot_detailed_prompt_1_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4o: tot prompt 2 average\")\n",
    "calculate_success_rate([gpt4o_tot_prompt_2_results, gpt4o_tot_detailed_prompt_2_results])\n",
    "calculate_average_score(gpt4o_tot_prompt_2_scores + gpt4o_tot_detailed_prompt_2_scores)\n",
    "\n",
    "print(\"GPT-4o: tot prompt 2\")\n",
    "calculate_success_rate(gpt4o_tot_prompt_2_results)\n",
    "calculate_average_score(gpt4o_tot_prompt_2_scores)\n",
    "\n",
    "print(\"GPT-4o: tot detailed prompt 2\")\n",
    "calculate_success_rate(gpt4o_tot_detailed_prompt_2_results)\n",
    "calculate_average_score(gpt4o_tot_detailed_prompt_2_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4o: tot prompt 3 average\")\n",
    "calculate_success_rate([gpt4o_tot_prompt_3_results, gpt4o_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt4o_tot_prompt_3_scores + gpt4o_tot_detailed_prompt_3_scores)\n",
    "\n",
    "print(\"GPT-4o: tot prompt 3\")\n",
    "calculate_success_rate(gpt4o_tot_prompt_3_results)\n",
    "calculate_average_score(gpt4o_tot_prompt_3_scores)\n",
    "\n",
    "print(\"GPT-4o: tot detailed prompt 3\")\n",
    "calculate_success_rate(gpt4o_tot_detailed_prompt_3_results)\n",
    "calculate_average_score(gpt4o_tot_detailed_prompt_3_scores)\n",
    "\n",
    "####\n",
    "# GPT-4o detailed average\n",
    "print(\"GPT-4o detailed average\")\n",
    "calculate_success_rate([gpt4o_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt4o_zs_role_system_prompt_detailed_results,\n",
    "                        gpt4o_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt4o_cot_human_prompt_detailed_results,\n",
    "                        gpt4o_cot_ape_prompt_detailed_results,\n",
    "                        gpt4o_meta_system_prompt_detailed_results,\n",
    "                        gpt4o_meta_meta_prompt_detailed_results,\n",
    "                        gpt4o_tot_detailed_prompt_1_results,\n",
    "                        gpt4o_tot_detailed_prompt_2_results,\n",
    "                        gpt4o_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt4o_zs_baseline_system_prompt_detailed_scores + gpt4o_zs_role_system_prompt_detailed_scores + gpt4o_zs_role_best_system_prompt_detailed_scores + gpt4o_cot_human_prompt_detailed_scores + gpt4o_cot_ape_prompt_detailed_scores + gpt4o_meta_system_prompt_detailed_scores + gpt4o_meta_meta_prompt_detailed_scores + gpt4o_tot_detailed_prompt_1_scores + gpt4o_tot_detailed_prompt_2_scores + gpt4o_tot_detailed_prompt_3_scores)\n",
    "\n",
    "####\n",
    "# GPT-4o not detailed average\n",
    "print(\"GPT-4o not detailed average\")\n",
    "calculate_success_rate([gpt4o_zs_baseline_system_prompt_results,\n",
    "                        gpt4o_zs_role_system_prompt_results,\n",
    "                        gpt4o_zs_role_best_system_prompt_results,\n",
    "                        gpt4o_cot_human_prompt_results,\n",
    "                        gpt4o_cot_ape_prompt_results,\n",
    "                        gpt4o_meta_system_prompt_results,\n",
    "                        gpt4o_meta_meta_prompt_results,\n",
    "                        gpt4o_tot_prompt_1_results,\n",
    "                        gpt4o_tot_prompt_2_results,\n",
    "                        gpt4o_tot_prompt_3_results])\n",
    "calculate_average_score(gpt4o_zs_baseline_system_prompt_scores + gpt4o_zs_role_system_prompt_scores + gpt4o_zs_role_best_system_prompt_scores + gpt4o_cot_human_prompt_scores + gpt4o_cot_ape_prompt_scores + gpt4o_meta_system_prompt_scores + gpt4o_meta_meta_prompt_scores + gpt4o_tot_prompt_1_scores + gpt4o_tot_prompt_2_scores + gpt4o_tot_prompt_3_scores)\n",
    "\n",
    "####\n",
    "# GPT-4o zero shot\n",
    "print(\"GPT-4o zero-shot average\")\n",
    "calculate_success_rate([gpt4o_zs_baseline_system_prompt_results,\n",
    "                        gpt4o_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt4o_zs_role_system_prompt_results,\n",
    "                        gpt4o_zs_role_system_prompt_detailed_scores,\n",
    "                        gpt4o_zs_role_best_system_prompt_results,\n",
    "                        gpt4o_zs_role_best_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt4o_zs_baseline_system_prompt_scores + gpt4o_zs_baseline_system_prompt_detailed_scores + gpt4o_zs_role_system_prompt_scores + gpt4o_zs_role_system_prompt_detailed_scores + gpt4o_zs_role_best_system_prompt_scores + gpt4o_zs_role_best_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4o cot average\")\n",
    "calculate_success_rate([gpt4o_cot_human_prompt_results,\n",
    "                        gpt4o_cot_human_prompt_detailed_results,\n",
    "                        gpt4o_cot_ape_prompt_results,\n",
    "                        gpt4o_cot_ape_prompt_detailed_results])\n",
    "calculate_average_score(gpt4o_cot_human_prompt_scores + gpt4o_cot_human_prompt_detailed_scores + gpt4o_cot_ape_prompt_scores + gpt4o_cot_ape_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4o Meta average\")\n",
    "calculate_success_rate([gpt4o_meta_system_prompt_results,\n",
    "                        gpt4o_meta_system_prompt_detailed_results,\n",
    "                        gpt4o_meta_meta_prompt_results,\n",
    "                        gpt4o_meta_meta_prompt_detailed_results])\n",
    "calculate_average_score(gpt4o_meta_system_prompt_scores + gpt4o_meta_system_prompt_detailed_scores + gpt4o_meta_meta_prompt_scores + gpt4o_meta_meta_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4o tot average\")\n",
    "calculate_success_rate([gpt4o_tot_prompt_1_results,\n",
    "                        gpt4o_tot_detailed_prompt_1_results,\n",
    "                        gpt4o_tot_prompt_2_results,\n",
    "                        gpt4o_tot_detailed_prompt_2_results,\n",
    "                        gpt4o_tot_prompt_3_results,\n",
    "                        gpt4o_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt4o_tot_prompt_1_scores + gpt4o_tot_detailed_prompt_1_scores + gpt4o_tot_prompt_2_scores + gpt4o_tot_detailed_prompt_2_scores + gpt4o_tot_prompt_3_scores + gpt4o_tot_detailed_prompt_3_scores)\n",
    "\n",
    "# GPT-4o Totale average\n",
    "print(\"GPT-4o Totale average\")\n",
    "calculate_success_rate([gpt4o_zs_baseline_system_prompt_results,\n",
    "                        gpt4o_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt4o_zs_role_system_prompt_results,\n",
    "                        gpt4o_zs_role_system_prompt_detailed_scores,\n",
    "                        gpt4o_zs_role_best_system_prompt_results,\n",
    "                        gpt4o_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt4o_cot_human_prompt_results,\n",
    "                        gpt4o_cot_human_prompt_detailed_results,\n",
    "                        gpt4o_cot_ape_prompt_results,\n",
    "                        gpt4o_cot_ape_prompt_detailed_results,\n",
    "                        gpt4o_meta_system_prompt_results,\n",
    "                        gpt4o_meta_system_prompt_detailed_results,\n",
    "                        gpt4o_meta_meta_prompt_results,\n",
    "                        gpt4o_meta_meta_prompt_detailed_results,\n",
    "                        gpt4o_tot_prompt_1_results,\n",
    "                        gpt4o_tot_detailed_prompt_1_results,\n",
    "                        gpt4o_tot_prompt_2_results,\n",
    "                        gpt4o_tot_detailed_prompt_2_results,\n",
    "                        gpt4o_tot_prompt_3_results,\n",
    "                        gpt4o_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt4o_zs_baseline_system_prompt_scores + gpt4o_zs_baseline_system_prompt_detailed_scores + gpt4o_zs_role_system_prompt_scores + gpt4o_zs_role_system_prompt_detailed_scores + gpt4o_zs_role_best_system_prompt_scores + gpt4o_zs_role_best_system_prompt_detailed_scores + gpt4o_cot_human_prompt_scores + gpt4o_cot_human_prompt_detailed_scores + gpt4o_cot_ape_prompt_scores + gpt4o_cot_ape_prompt_detailed_scores + gpt4o_meta_system_prompt_scores + gpt4o_meta_system_prompt_detailed_scores + gpt4o_meta_meta_prompt_scores + gpt4o_meta_meta_prompt_detailed_scores + gpt4o_tot_prompt_1_scores + gpt4o_tot_detailed_prompt_1_scores + gpt4o_tot_prompt_2_scores + gpt4o_tot_detailed_prompt_2_scores + gpt4o_tot_prompt_3_scores + gpt4o_tot_detailed_prompt_3_scores)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT-4: zero-shot baseline average\n",
      "Succes rate: 0.39\n",
      "Avg score: 40.38461538461539\n",
      "GPT-4: zero-shot baseline system prompt\n",
      "Succes rate: 0.2\n",
      "Avg score: 35.5\n",
      "GPT-4: zero-shot baseline system prompt detailed\n",
      "Succes rate: 0.58\n",
      "Avg score: 42.06896551724138\n",
      "GPT-4: zero-shot role system average\n",
      "Succes rate: 0.41\n",
      "Avg score: 41.170731707317074\n",
      "GPT-4: zero-shot role system prompt\n",
      "Succes rate: 0.12\n",
      "Avg score: 37.0\n",
      "GPT-4: zero-shot role system prompt detailed\n",
      "Succes rate: 0.7\n",
      "Avg score: 41.885714285714286\n",
      "GPT-4: zero-shot role best system average\n",
      "Succes rate: 0.41\n",
      "Avg score: 41.65853658536585\n",
      "GPT-4: zero-shot role best system prompt\n",
      "Succes rate: 0.1\n",
      "Avg score: 37.8\n",
      "GPT-4: zero-shot role best system prompt detailed\n",
      "Succes rate: 0.72\n",
      "Avg score: 42.19444444444444\n",
      "GPT-4: cot human average\n",
      "Succes rate: 0.46\n",
      "Avg score: 40.391304347826086\n",
      "GPT-4: cot human prompt\n",
      "Succes rate: 0.3\n",
      "Avg score: 37.13333333333333\n",
      "GPT-4: cot human prompt detailed\n",
      "Succes rate: 0.62\n",
      "Avg score: 41.96774193548387\n",
      "GPT-4: cot ape average\n",
      "Succes rate: 0.44\n",
      "Avg score: 39.88636363636363\n",
      "GPT-4: cot ape prompt\n",
      "Succes rate: 0.36\n",
      "Avg score: 37.5\n",
      "GPT-4: cot ape prompt detailed\n",
      "Succes rate: 0.52\n",
      "Avg score: 41.53846153846154\n",
      "GPT-4: Meta system average\n",
      "Succes rate: 0.43\n",
      "Avg score: 40.55813953488372\n",
      "GPT-4: Meta system prompt\n",
      "Succes rate: 0.16\n",
      "Avg score: 39.375\n",
      "GPT-4: Meta system prompt detailed\n",
      "Succes rate: 0.7\n",
      "Avg score: 40.82857142857143\n",
      "GPT-4: Meta meta average\n",
      "Succes rate: 0.09\n",
      "Avg score: 50.333333333333336\n",
      "GPT-4: Meta meta prompt\n",
      "Succes rate: 0.02\n",
      "Avg score: 43.0\n",
      "GPT-4: Meta meta prompt detailed\n",
      "Succes rate: 0.16\n",
      "Avg score: 51.25\n",
      "GPT-4: tot prompt 1 average\n",
      "Succes rate: 0.31\n",
      "Avg score: 39.225806451612904\n",
      "GPT-4: tot prompt 1\n",
      "Succes rate: 0.26\n",
      "Avg score: 35.30769230769231\n",
      "GPT-4: tot detailed prompt 1\n",
      "Succes rate: 0.36\n",
      "Avg score: 42.05555555555556\n",
      "GPT-4: tot prompt 2 average\n",
      "Succes rate: 0.11\n",
      "Avg score: 40.72727272727273\n",
      "GPT-4: tot prompt 2\n",
      "Succes rate: 0.06\n",
      "Avg score: 36.666666666666664\n",
      "GPT-4: tot detailed prompt 2\n",
      "Succes rate: 0.16\n",
      "Avg score: 42.25\n",
      "GPT-4: tot prompt 3 average\n",
      "Succes rate: 0.02\n",
      "Avg score: 39.5\n",
      "GPT-4: tot prompt 3\n",
      "Succes rate: 0.02\n",
      "Avg score: 35.0\n",
      "GPT-4: tot detailed prompt 3\n",
      "Succes rate: 0.02\n",
      "Avg score: 44.0\n",
      "GPT-4 detailed average\n",
      "Succes rate: 0.454\n",
      "Avg score: 42.13215859030837\n",
      "GPT-4 not detailed average\n",
      "Succes rate: 0.16\n",
      "Avg score: 37.0\n",
      "GPT-4 zero-shot average\n",
      "Succes rate: 0.4033333333333333\n",
      "Avg score: 41.082644628099175\n",
      "GPT-4 cot average\n",
      "Succes rate: 0.45\n",
      "Avg score: 40.144444444444446\n",
      "GPT-4 Meta average\n",
      "Succes rate: 0.26\n",
      "Avg score: 42.25\n",
      "GPT-4 tot average\n",
      "Succes rate: 0.14666666666666667\n",
      "Avg score: 39.61363636363637\n",
      "GPT-4 Totale average\n",
      "Succes rate: 0.307\n",
      "Avg score: 40.79478827361564\n"
     ]
    }
   ],
   "source": [
    "# GPT-4\n",
    "print(\"GPT-4: zero-shot baseline average\")\n",
    "calculate_success_rate([gpt4_zs_baseline_system_prompt_results, gpt4_zs_baseline_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt4_zs_baseline_system_prompt_scores + gpt4_zs_baseline_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4: zero-shot baseline system prompt\")\n",
    "calculate_success_rate(gpt4_zs_baseline_system_prompt_results)\n",
    "calculate_average_score(gpt4_zs_baseline_system_prompt_scores)\n",
    "\n",
    "print(\"GPT-4: zero-shot baseline system prompt detailed\")\n",
    "calculate_success_rate(gpt4_zs_baseline_system_prompt_detailed_results)\n",
    "calculate_average_score(gpt4_zs_baseline_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4: zero-shot role system average\")\n",
    "calculate_success_rate([gpt4_zs_role_system_prompt_results, gpt4_zs_role_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt4_zs_role_system_prompt_scores + gpt4_zs_role_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4: zero-shot role system prompt\")\n",
    "calculate_success_rate(gpt4_zs_role_system_prompt_results)\n",
    "calculate_average_score(gpt4_zs_role_system_prompt_scores)\n",
    "\n",
    "print(\"GPT-4: zero-shot role system prompt detailed\")\n",
    "calculate_success_rate(gpt4_zs_role_system_prompt_detailed_results)\n",
    "calculate_average_score(gpt4_zs_role_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4: zero-shot role best system average\")\n",
    "calculate_success_rate([gpt4_zs_role_best_system_prompt_results, gpt4_zs_role_best_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt4_zs_role_best_system_prompt_scores + gpt4_zs_role_best_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4: zero-shot role best system prompt\")\n",
    "calculate_success_rate(gpt4_zs_role_best_system_prompt_results)\n",
    "calculate_average_score(gpt4_zs_role_best_system_prompt_scores)\n",
    "\n",
    "print(\"GPT-4: zero-shot role best system prompt detailed\")\n",
    "calculate_success_rate(gpt4_zs_role_best_system_prompt_detailed_results)\n",
    "calculate_average_score(gpt4_zs_role_best_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4: cot human average\")\n",
    "calculate_success_rate([gpt4_cot_human_prompt_results, gpt4_cot_human_prompt_detailed_results])\n",
    "calculate_average_score(gpt4_cot_human_prompt_scores + gpt4_cot_human_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4: cot human prompt\")\n",
    "calculate_success_rate(gpt4_cot_human_prompt_results)\n",
    "calculate_average_score(gpt4_cot_human_prompt_scores)\n",
    "\n",
    "print(\"GPT-4: cot human prompt detailed\")\n",
    "calculate_success_rate(gpt4_cot_human_prompt_detailed_results)\n",
    "calculate_average_score(gpt4_cot_human_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4: cot ape average\")\n",
    "calculate_success_rate([gpt4_cot_ape_prompt_results, gpt4_cot_ape_prompt_detailed_results])\n",
    "calculate_average_score(gpt4_cot_ape_prompt_scores + gpt4_cot_ape_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4: cot ape prompt\")\n",
    "calculate_success_rate(gpt4_cot_ape_prompt_results)\n",
    "calculate_average_score(gpt4_cot_ape_prompt_scores)\n",
    "\n",
    "print(\"GPT-4: cot ape prompt detailed\")\n",
    "calculate_success_rate(gpt4_cot_ape_prompt_detailed_results)\n",
    "calculate_average_score(gpt4_cot_ape_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4: Meta system average\")\n",
    "calculate_success_rate([gpt4_meta_system_prompt_results, gpt4_meta_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt4_meta_system_prompt_scores + gpt4_meta_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4: Meta system prompt\")\n",
    "calculate_success_rate(gpt4_meta_system_prompt_results)\n",
    "calculate_average_score(gpt4_meta_system_prompt_scores)\n",
    "\n",
    "print(\"GPT-4: Meta system prompt detailed\")\n",
    "calculate_success_rate(gpt4_meta_system_prompt_detailed_results)\n",
    "calculate_average_score(gpt4_meta_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4: Meta meta average\")\n",
    "calculate_success_rate([gpt4_meta_meta_prompt_results, gpt4_meta_meta_prompt_detailed_results])\n",
    "calculate_average_score(gpt4_meta_meta_prompt_scores + gpt4_meta_meta_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4: Meta meta prompt\")\n",
    "calculate_success_rate(gpt4_meta_meta_prompt_results)\n",
    "calculate_average_score(gpt4_meta_meta_prompt_scores)\n",
    "\n",
    "print(\"GPT-4: Meta meta prompt detailed\")\n",
    "calculate_success_rate(gpt4_meta_meta_prompt_detailed_results)\n",
    "calculate_average_score(gpt4_meta_meta_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4: tot prompt 1 average\")\n",
    "calculate_success_rate([gpt4_tot_prompt_1_results, gpt4_tot_detailed_prompt_1_results])\n",
    "calculate_average_score(gpt4_tot_prompt_1_scores + gpt4_tot_detailed_prompt_1_scores)\n",
    "\n",
    "print(\"GPT-4: tot prompt 1\")\n",
    "calculate_success_rate(gpt4_tot_prompt_1_results)\n",
    "calculate_average_score(gpt4_tot_prompt_1_scores)\n",
    "\n",
    "print(\"GPT-4: tot detailed prompt 1\")\n",
    "calculate_success_rate(gpt4_tot_detailed_prompt_1_results)\n",
    "calculate_average_score(gpt4_tot_detailed_prompt_1_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4: tot prompt 2 average\")\n",
    "calculate_success_rate([gpt4_tot_prompt_2_results, gpt4_tot_detailed_prompt_2_results])\n",
    "calculate_average_score(gpt4_tot_prompt_2_scores + gpt4_tot_detailed_prompt_2_scores)\n",
    "\n",
    "print(\"GPT-4: tot prompt 2\")\n",
    "calculate_success_rate(gpt4_tot_prompt_2_results)\n",
    "calculate_average_score(gpt4_tot_prompt_2_scores)\n",
    "\n",
    "print(\"GPT-4: tot detailed prompt 2\")\n",
    "calculate_success_rate(gpt4_tot_detailed_prompt_2_results)\n",
    "calculate_average_score(gpt4_tot_detailed_prompt_2_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-4: tot prompt 3 average\")\n",
    "calculate_success_rate([gpt4_tot_prompt_3_results, gpt4_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt4_tot_prompt_3_scores + gpt4_tot_detailed_prompt_3_scores)\n",
    "\n",
    "print(\"GPT-4: tot prompt 3\")\n",
    "calculate_success_rate(gpt4_tot_prompt_3_results)\n",
    "calculate_average_score(gpt4_tot_prompt_3_scores)\n",
    "\n",
    "print(\"GPT-4: tot detailed prompt 3\")\n",
    "calculate_success_rate(gpt4_tot_detailed_prompt_3_results)\n",
    "calculate_average_score(gpt4_tot_detailed_prompt_3_scores)\n",
    "\n",
    "####\n",
    "# GPT-4 detailed average\n",
    "print(\"GPT-4 detailed average\")\n",
    "calculate_success_rate([gpt4_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt4_zs_role_system_prompt_detailed_results,\n",
    "                        gpt4_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt4_cot_human_prompt_detailed_results,\n",
    "                        gpt4_cot_ape_prompt_detailed_results,\n",
    "                        gpt4_meta_system_prompt_detailed_results,\n",
    "                        gpt4_meta_meta_prompt_detailed_results,\n",
    "                        gpt4_tot_detailed_prompt_1_results,\n",
    "                        gpt4_tot_detailed_prompt_2_results,\n",
    "                        gpt4_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt4_zs_baseline_system_prompt_detailed_scores + gpt4_zs_role_system_prompt_detailed_scores + gpt4_zs_role_best_system_prompt_detailed_scores + gpt4_cot_human_prompt_detailed_scores + gpt4_cot_ape_prompt_detailed_scores + gpt4_meta_system_prompt_detailed_scores + gpt4_meta_meta_prompt_detailed_scores + gpt4_tot_detailed_prompt_1_scores + gpt4_tot_detailed_prompt_2_scores + gpt4_tot_detailed_prompt_3_scores)\n",
    "\n",
    "# GPT-4 not detailed average\n",
    "print(\"GPT-4 not detailed average\")\n",
    "calculate_success_rate([gpt4_zs_baseline_system_prompt_results,\n",
    "                        gpt4_zs_role_system_prompt_results,\n",
    "                        gpt4_zs_role_best_system_prompt_results,\n",
    "                        gpt4_cot_human_prompt_results,\n",
    "                        gpt4_cot_ape_prompt_results,\n",
    "                        gpt4_meta_system_prompt_results,\n",
    "                        gpt4_meta_meta_prompt_results,\n",
    "                        gpt4_tot_prompt_1_results,\n",
    "                        gpt4_tot_prompt_2_results,\n",
    "                        gpt4_tot_prompt_3_results])\n",
    "calculate_average_score(gpt4_zs_baseline_system_prompt_scores + gpt4_zs_role_system_prompt_scores + gpt4_zs_role_best_system_prompt_scores + gpt4_cot_human_prompt_scores + gpt4_cot_ape_prompt_scores + gpt4_meta_system_prompt_scores + gpt4_meta_meta_prompt_scores + gpt4_tot_prompt_1_scores + gpt4_tot_prompt_2_scores + gpt4_tot_prompt_3_scores)\n",
    "\n",
    "####\n",
    "# GPT-4 zero shot\n",
    "print(\"GPT-4 zero-shot average\")\n",
    "calculate_success_rate([gpt4_zs_baseline_system_prompt_results,\n",
    "                        gpt4_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt4_zs_role_system_prompt_results,\n",
    "                        gpt4_zs_role_system_prompt_detailed_results,\n",
    "                        gpt4_zs_role_best_system_prompt_results,\n",
    "                        gpt4_zs_role_best_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt4_zs_baseline_system_prompt_scores + gpt4_zs_baseline_system_prompt_detailed_scores + gpt4_zs_role_system_prompt_scores + gpt4_zs_role_system_prompt_detailed_scores + gpt4_zs_role_best_system_prompt_scores + gpt4_zs_role_best_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4 cot average\")\n",
    "calculate_success_rate([gpt4_cot_human_prompt_results,\n",
    "                        gpt4_cot_human_prompt_detailed_results,\n",
    "                        gpt4_cot_ape_prompt_results,\n",
    "                        gpt4_cot_ape_prompt_detailed_results])\n",
    "calculate_average_score(gpt4_cot_human_prompt_scores + gpt4_cot_human_prompt_detailed_scores + gpt4_cot_ape_prompt_scores + gpt4_cot_ape_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4 Meta average\")\n",
    "calculate_success_rate([gpt4_meta_system_prompt_results,\n",
    "                        gpt4_meta_system_prompt_detailed_results,\n",
    "                        gpt4_meta_meta_prompt_results,\n",
    "                        gpt4_meta_meta_prompt_detailed_results])\n",
    "\n",
    "calculate_average_score(gpt4_meta_system_prompt_scores + gpt4_meta_system_prompt_detailed_scores + gpt4_meta_meta_prompt_scores + gpt4_meta_meta_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-4 tot average\")\n",
    "calculate_success_rate([gpt4_tot_prompt_1_results,\n",
    "                        gpt4_tot_detailed_prompt_1_results,\n",
    "                        gpt4_tot_prompt_2_results,\n",
    "                        gpt4_tot_detailed_prompt_2_results,\n",
    "                        gpt4_tot_prompt_3_results,\n",
    "                        gpt4_tot_detailed_prompt_3_results])\n",
    "\n",
    "calculate_average_score(gpt4_tot_prompt_1_scores + gpt4_tot_detailed_prompt_1_scores + gpt4_tot_prompt_2_scores + gpt4_tot_detailed_prompt_2_scores + gpt4_tot_prompt_3_scores + gpt4_tot_detailed_prompt_3_scores)\n",
    "\n",
    "# GPT-4 Totale average\n",
    "print(\"GPT-4 Totale average\")\n",
    "calculate_success_rate([gpt4_zs_baseline_system_prompt_results,\n",
    "                        gpt4_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt4_zs_role_system_prompt_results,\n",
    "                        gpt4_zs_role_system_prompt_detailed_results,\n",
    "                        gpt4_zs_role_best_system_prompt_results,\n",
    "                        gpt4_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt4_cot_human_prompt_results,\n",
    "                        gpt4_cot_human_prompt_detailed_results,\n",
    "                        gpt4_cot_ape_prompt_results,\n",
    "                        gpt4_cot_ape_prompt_detailed_results,\n",
    "                        gpt4_meta_system_prompt_results,\n",
    "                        gpt4_meta_system_prompt_detailed_results,\n",
    "                        gpt4_meta_meta_prompt_results,\n",
    "                        gpt4_meta_meta_prompt_detailed_results,\n",
    "                        gpt4_tot_prompt_1_results,\n",
    "                        gpt4_tot_detailed_prompt_1_results,\n",
    "                        gpt4_tot_prompt_2_results,\n",
    "                        gpt4_tot_detailed_prompt_2_results,\n",
    "                        gpt4_tot_prompt_3_results,\n",
    "                        gpt4_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt4_zs_baseline_system_prompt_scores + gpt4_zs_baseline_system_prompt_detailed_scores + gpt4_zs_role_system_prompt_scores + gpt4_zs_role_system_prompt_detailed_scores + gpt4_zs_role_best_system_prompt_scores + gpt4_zs_role_best_system_prompt_detailed_scores + gpt4_cot_human_prompt_scores + gpt4_cot_human_prompt_detailed_scores + gpt4_cot_ape_prompt_scores + gpt4_cot_ape_prompt_detailed_scores + gpt4_meta_system_prompt_scores + gpt4_meta_system_prompt_detailed_scores + gpt4_meta_meta_prompt_scores + gpt4_meta_meta_prompt_detailed_scores + gpt4_tot_prompt_1_scores + gpt4_tot_detailed_prompt_1_scores + gpt4_tot_prompt_2_scores + gpt4_tot_detailed_prompt_2_scores + gpt4_tot_prompt_3_scores + gpt4_tot_detailed_prompt_3_scores)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT-3.5: zero-shot baseline average\n",
      "Succes rate: 0.06\n",
      "Avg score: 36.166666666666664\n",
      "GPT-3.5: zero-shot baseline system prompt\n",
      "Succes rate: 0.06\n",
      "Avg score: 34.0\n",
      "GPT-3.5: zero-shot baseline system prompt detailed\n",
      "Succes rate: 0.06\n",
      "Avg score: 38.333333333333336\n",
      "GPT-3.5: zero-shot role system average\n",
      "Succes rate: 0.04\n",
      "Avg score: 37.5\n",
      "GPT-3.5: zero-shot role system prompt\n",
      "Succes rate: 0.02\n",
      "Avg score: 35.0\n",
      "GPT-3.5: zero-shot role system prompt detailed\n",
      "Succes rate: 0.06\n",
      "Avg score: 38.333333333333336\n",
      "GPT-3.5: zero-shot role best system average\n",
      "Succes rate: 0.01\n",
      "Avg score: 41.0\n",
      "GPT-3.5: zero-shot role best system prompt\n",
      "Succes rate: 0.0\n",
      "Avg score: 0\n",
      "GPT-3.5: zero-shot role best system prompt detailed\n",
      "Succes rate: 0.02\n",
      "Avg score: 41.0\n",
      "GPT-3.5: cot human average\n",
      "Succes rate: 0.11\n",
      "Avg score: 34.81818181818182\n",
      "GPT-3.5: cot human prompt\n",
      "Succes rate: 0.18\n",
      "Avg score: 34.22222222222222\n",
      "GPT-3.5: cot human prompt detailed\n",
      "Succes rate: 0.04\n",
      "Avg score: 37.5\n",
      "GPT-3.5: cot ape average\n",
      "Succes rate: 0.17\n",
      "Avg score: 34.0\n",
      "GPT-3.5: cot ape prompt\n",
      "Succes rate: 0.2\n",
      "Avg score: 32.3\n",
      "GPT-3.5: cot ape prompt detailed\n",
      "Succes rate: 0.14\n",
      "Avg score: 36.42857142857143\n",
      "GPT-3.5: Meta system average\n",
      "Succes rate: 0.04\n",
      "Avg score: 36.0\n",
      "GPT-3.5: Meta system prompt\n",
      "Succes rate: 0.06\n",
      "Avg score: 36.0\n",
      "GPT-3.5: Meta system prompt detailed\n",
      "Succes rate: 0.02\n",
      "Avg score: 36.0\n",
      "GPT-3.5: Meta meta average\n",
      "Succes rate: 0.02\n",
      "Avg score: 42.5\n",
      "GPT-3.5: Meta meta prompt\n",
      "Succes rate: 0.04\n",
      "Avg score: 42.5\n",
      "GPT-3.5: Meta meta prompt detailed\n",
      "Succes rate: 0.0\n",
      "Avg score: 0\n",
      "GPT-3.5: tot prompt 1 average\n",
      "Succes rate: 0.0\n",
      "Avg score: 0\n",
      "GPT-3.5: tot prompt 1\n",
      "Succes rate: 0.0\n",
      "Avg score: 0\n",
      "GPT-3.5: tot detailed prompt 1\n",
      "Succes rate: 0.0\n",
      "Avg score: 0\n",
      "GPT-3.5: tot prompt 2 average\n",
      "Succes rate: 0.01\n",
      "Avg score: 36.0\n",
      "GPT-3.5: tot prompt 2\n",
      "Succes rate: 0.0\n",
      "Avg score: 0\n",
      "GPT-3.5: tot detailed prompt 2\n",
      "Succes rate: 0.02\n",
      "Avg score: 36.0\n",
      "GPT-3.5: tot prompt 3 average\n",
      "Succes rate: 0.0\n",
      "Avg score: 0\n",
      "GPT-3.5: tot prompt 3\n",
      "Succes rate: 0.0\n",
      "Avg score: 0\n",
      "GPT-3.5: tot detailed prompt 3\n",
      "Succes rate: 0.0\n",
      "Avg score: 0\n",
      "GPT-3.5 detailed average\n",
      "Succes rate: 0.03\n",
      "Avg score: 37.2\n",
      "GPT-3.5 not detailed average\n",
      "Succes rate: 0.056\n",
      "Avg score: 34.32142857142857\n",
      "GPT-3.5 zero-shot average\n",
      "Succes rate: 0.02666666666666667\n",
      "Avg score: 36.625\n",
      "GPT-3.5 cot average\n",
      "Succes rate: 0.14\n",
      "Avg score: 34.32142857142857\n",
      "GPT-3.5 Meta average\n",
      "Succes rate: 0.03\n",
      "Avg score: 38.166666666666664\n",
      "GPT-3.5 tot average\n",
      "Succes rate: 0.0033333333333333335\n",
      "Avg score: 36.0\n",
      "GPT-3.5 Totale average\n",
      "Succes rate: 0.043\n",
      "Avg score: 35.325581395348834\n"
     ]
    }
   ],
   "source": [
    "# GPT-3.5\n",
    "print(\"GPT-3.5: zero-shot baseline average\")\n",
    "calculate_success_rate([gpt3_5_zs_baseline_system_prompt_results, gpt3_5_zs_role_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_zs_baseline_system_prompt_scores + gpt3_5_zs_role_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-3.5: zero-shot baseline system prompt\")\n",
    "calculate_success_rate(gpt3_5_zs_baseline_system_prompt_results)\n",
    "calculate_average_score(gpt3_5_zs_baseline_system_prompt_scores)\n",
    "\n",
    "print(\"GPT-3.5: zero-shot baseline system prompt detailed\")\n",
    "calculate_success_rate(gpt3_5_zs_role_system_prompt_detailed_results)\n",
    "calculate_average_score(gpt3_5_zs_role_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-3.5: zero-shot role system average\")\n",
    "calculate_success_rate([gpt3_5_zs_role_system_prompt_results, gpt3_5_zs_role_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_zs_role_system_prompt_scores + gpt3_5_zs_role_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-3.5: zero-shot role system prompt\")\n",
    "calculate_success_rate(gpt3_5_zs_role_system_prompt_results)\n",
    "calculate_average_score(gpt3_5_zs_role_system_prompt_scores)\n",
    "\n",
    "print(\"GPT-3.5: zero-shot role system prompt detailed\")\n",
    "calculate_success_rate(gpt3_5_zs_role_system_prompt_detailed_results)\n",
    "calculate_average_score(gpt3_5_zs_role_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-3.5: zero-shot role best system average\")\n",
    "calculate_success_rate([gpt3_5_zs_role_best_system_prompt_results, gpt3_5_zs_role_best_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_zs_role_best_system_prompt_scores + gpt3_5_zs_role_best_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-3.5: zero-shot role best system prompt\")\n",
    "calculate_success_rate(gpt3_5_zs_role_best_system_prompt_results)\n",
    "calculate_average_score(gpt3_5_zs_role_best_system_prompt_scores)\n",
    "\n",
    "print(\"GPT-3.5: zero-shot role best system prompt detailed\")\n",
    "calculate_success_rate(gpt3_5_zs_role_best_system_prompt_detailed_results)\n",
    "calculate_average_score(gpt3_5_zs_role_best_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-3.5: cot human average\")\n",
    "calculate_success_rate([gpt3_5_cot_human_prompt_results, gpt3_5_cot_human_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_cot_human_prompt_scores + gpt3_5_cot_human_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-3.5: cot human prompt\")\n",
    "calculate_success_rate(gpt3_5_cot_human_prompt_results)\n",
    "calculate_average_score(gpt3_5_cot_human_prompt_scores)\n",
    "\n",
    "print(\"GPT-3.5: cot human prompt detailed\")\n",
    "calculate_success_rate(gpt3_5_cot_human_prompt_detailed_results)\n",
    "calculate_average_score(gpt3_5_cot_human_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-3.5: cot ape average\")\n",
    "calculate_success_rate([gpt3_5_cot_ape_prompt_results, gpt3_5_cot_ape_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_cot_ape_prompt_scores + gpt3_5_cot_ape_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-3.5: cot ape prompt\")\n",
    "calculate_success_rate(gpt3_5_cot_ape_prompt_results)\n",
    "calculate_average_score(gpt3_5_cot_ape_prompt_scores)\n",
    "\n",
    "print(\"GPT-3.5: cot ape prompt detailed\")\n",
    "calculate_success_rate(gpt3_5_cot_ape_prompt_detailed_results)\n",
    "calculate_average_score(gpt3_5_cot_ape_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-3.5: Meta system average\")\n",
    "calculate_success_rate([gpt3_5_meta_system_prompt_results, gpt3_5_meta_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_meta_system_prompt_scores + gpt3_5_meta_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-3.5: Meta system prompt\")\n",
    "calculate_success_rate(gpt3_5_meta_system_prompt_results)\n",
    "calculate_average_score(gpt3_5_meta_system_prompt_scores)\n",
    "\n",
    "print(\"GPT-3.5: Meta system prompt detailed\")\n",
    "calculate_success_rate(gpt3_5_meta_system_prompt_detailed_results)\n",
    "calculate_average_score(gpt3_5_meta_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-3.5: Meta meta average\")\n",
    "calculate_success_rate([gpt3_5_meta_meta_prompt_results, gpt3_5_meta_meta_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_meta_meta_prompt_scores + gpt3_5_meta_meta_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-3.5: Meta meta prompt\")\n",
    "calculate_success_rate(gpt3_5_meta_meta_prompt_results)\n",
    "calculate_average_score(gpt3_5_meta_meta_prompt_scores)\n",
    "\n",
    "print(\"GPT-3.5: Meta meta prompt detailed\")\n",
    "calculate_success_rate(gpt3_5_meta_meta_prompt_detailed_results)\n",
    "calculate_average_score(gpt3_5_meta_meta_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-3.5: tot prompt 1 average\")\n",
    "calculate_success_rate([gpt3_5_tot_prompt_1_results, gpt3_5_tot_detailed_prompt_1_results])\n",
    "calculate_average_score(gpt3_5_tot_prompt_1_scores + gpt3_5_tot_detailed_prompt_1_scores)\n",
    "\n",
    "print(\"GPT-3.5: tot prompt 1\")\n",
    "calculate_success_rate(gpt3_5_tot_prompt_1_results)\n",
    "calculate_average_score(gpt3_5_tot_prompt_1_scores)\n",
    "\n",
    "print(\"GPT-3.5: tot detailed prompt 1\")\n",
    "calculate_success_rate(gpt3_5_tot_detailed_prompt_1_results)\n",
    "calculate_average_score(gpt3_5_tot_detailed_prompt_1_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-3.5: tot prompt 2 average\")\n",
    "calculate_success_rate([gpt3_5_tot_prompt_2_results, gpt3_5_tot_detailed_prompt_2_results])\n",
    "calculate_average_score(gpt3_5_tot_prompt_2_scores + gpt3_5_tot_detailed_prompt_2_scores)\n",
    "\n",
    "print(\"GPT-3.5: tot prompt 2\")\n",
    "calculate_success_rate(gpt3_5_tot_prompt_2_results)\n",
    "calculate_average_score(gpt3_5_tot_prompt_2_scores)\n",
    "\n",
    "print(\"GPT-3.5: tot detailed prompt 2\")\n",
    "calculate_success_rate(gpt3_5_tot_detailed_prompt_2_results)\n",
    "calculate_average_score(gpt3_5_tot_detailed_prompt_2_scores)\n",
    "\n",
    "####\n",
    "print(\"GPT-3.5: tot prompt 3 average\")\n",
    "calculate_success_rate([gpt3_5_tot_prompt_3_results, gpt3_5_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt3_5_tot_prompt_3_scores + gpt3_5_tot_detailed_prompt_3_scores)\n",
    "\n",
    "print(\"GPT-3.5: tot prompt 3\")\n",
    "calculate_success_rate(gpt3_5_tot_prompt_3_results)\n",
    "calculate_average_score(gpt3_5_tot_prompt_3_scores)\n",
    "\n",
    "print(\"GPT-3.5: tot detailed prompt 3\")\n",
    "calculate_success_rate(gpt3_5_tot_detailed_prompt_3_results)\n",
    "calculate_average_score(gpt3_5_tot_detailed_prompt_3_scores)\n",
    "\n",
    "####\n",
    "# GPT-3.5 detailed average\n",
    "print(\"GPT-3.5 detailed average\")\n",
    "calculate_success_rate([gpt3_5_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt3_5_zs_role_system_prompt_detailed_results,\n",
    "                        gpt3_5_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt3_5_cot_human_prompt_detailed_results,\n",
    "                        gpt3_5_cot_ape_prompt_detailed_results,\n",
    "                        gpt3_5_meta_system_prompt_detailed_results,\n",
    "                        gpt3_5_meta_meta_prompt_detailed_results,\n",
    "                        gpt3_5_tot_detailed_prompt_1_results,\n",
    "                        gpt3_5_tot_detailed_prompt_2_results,\n",
    "                        gpt3_5_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt3_5_zs_baseline_system_prompt_detailed_scores + gpt3_5_zs_role_system_prompt_detailed_scores + gpt3_5_zs_role_best_system_prompt_detailed_scores + gpt3_5_cot_human_prompt_detailed_scores + gpt3_5_cot_ape_prompt_detailed_scores + gpt3_5_meta_system_prompt_detailed_scores + gpt3_5_meta_meta_prompt_detailed_scores + gpt3_5_tot_detailed_prompt_1_scores + gpt3_5_tot_detailed_prompt_2_scores + gpt3_5_tot_detailed_prompt_3_scores)\n",
    "\n",
    "# GPT-3.5 not detailed average\n",
    "print(\"GPT-3.5 not detailed average\")\n",
    "calculate_success_rate([gpt3_5_zs_baseline_system_prompt_results,\n",
    "                        gpt3_5_zs_role_system_prompt_results,\n",
    "                        gpt3_5_zs_role_best_system_prompt_results,\n",
    "                        gpt3_5_cot_human_prompt_results,\n",
    "                        gpt3_5_cot_ape_prompt_results,\n",
    "                        gpt3_5_meta_system_prompt_results,\n",
    "                        gpt3_5_meta_meta_prompt_results,\n",
    "                        gpt3_5_tot_prompt_1_results,\n",
    "                        gpt3_5_tot_prompt_2_results,\n",
    "                        gpt3_5_tot_prompt_3_results])\n",
    "calculate_average_score(gpt3_5_zs_baseline_system_prompt_scores + gpt3_5_zs_role_system_prompt_scores + gpt3_5_zs_role_best_system_prompt_scores + gpt3_5_cot_human_prompt_scores + gpt3_5_cot_ape_prompt_scores + gpt3_5_meta_system_prompt_scores + gpt3_5_meta_meta_prompt_scores + gpt3_5_tot_prompt_1_scores + gpt3_5_tot_prompt_2_scores + gpt3_5_tot_prompt_3_scores)\n",
    "\n",
    "# GPT-3.5 zero shot\n",
    "print(\"GPT-3.5 zero-shot average\")\n",
    "calculate_success_rate([gpt3_5_zs_baseline_system_prompt_results,\n",
    "                        gpt3_5_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt3_5_zs_role_system_prompt_results,\n",
    "                        gpt3_5_zs_role_system_prompt_detailed_results,\n",
    "                        gpt3_5_zs_role_best_system_prompt_results,\n",
    "                        gpt3_5_zs_role_best_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_zs_baseline_system_prompt_scores + gpt3_5_zs_baseline_system_prompt_detailed_scores + gpt3_5_zs_role_system_prompt_scores + gpt3_5_zs_role_system_prompt_detailed_scores + gpt3_5_zs_role_best_system_prompt_scores + gpt3_5_zs_role_best_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-3.5 cot average\")\n",
    "calculate_success_rate([gpt3_5_cot_human_prompt_results,\n",
    "                        gpt3_5_cot_human_prompt_detailed_results,\n",
    "                        gpt3_5_cot_ape_prompt_results,\n",
    "                        gpt3_5_cot_ape_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_cot_human_prompt_scores + gpt3_5_cot_human_prompt_detailed_scores + gpt3_5_cot_ape_prompt_scores + gpt3_5_cot_ape_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-3.5 Meta average\")\n",
    "calculate_success_rate([gpt3_5_meta_system_prompt_results,\n",
    "                        gpt3_5_meta_system_prompt_detailed_results,\n",
    "                        gpt3_5_meta_meta_prompt_results,\n",
    "                        gpt3_5_meta_meta_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_meta_system_prompt_scores + gpt3_5_meta_system_prompt_detailed_scores + gpt3_5_meta_meta_prompt_scores + gpt3_5_meta_meta_prompt_detailed_scores)\n",
    "\n",
    "print(\"GPT-3.5 tot average\")\n",
    "calculate_success_rate([gpt3_5_tot_prompt_1_results,\n",
    "                        gpt3_5_tot_detailed_prompt_1_results,\n",
    "                        gpt3_5_tot_prompt_2_results,\n",
    "                        gpt3_5_tot_detailed_prompt_2_results,\n",
    "                        gpt3_5_tot_prompt_3_results,\n",
    "                        gpt3_5_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt3_5_tot_prompt_1_scores + gpt3_5_tot_detailed_prompt_1_scores + gpt3_5_tot_prompt_2_scores + gpt3_5_tot_detailed_prompt_2_scores + gpt3_5_tot_prompt_3_scores + gpt3_5_tot_detailed_prompt_3_scores)\n",
    "\n",
    "# GPT-3.5 Totale average\n",
    "print(\"GPT-3.5 Totale average\")\n",
    "calculate_success_rate([gpt3_5_zs_baseline_system_prompt_results,\n",
    "                        gpt3_5_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt3_5_zs_role_system_prompt_results,\n",
    "                        gpt3_5_zs_role_system_prompt_detailed_results,\n",
    "                        gpt3_5_zs_role_best_system_prompt_results,\n",
    "                        gpt3_5_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt3_5_cot_human_prompt_results,\n",
    "                        gpt3_5_cot_human_prompt_detailed_results,\n",
    "                        gpt3_5_cot_ape_prompt_results,\n",
    "                        gpt3_5_cot_ape_prompt_detailed_results,\n",
    "                        gpt3_5_meta_system_prompt_results,\n",
    "                        gpt3_5_meta_system_prompt_detailed_results,\n",
    "                        gpt3_5_meta_meta_prompt_results,\n",
    "                        gpt3_5_meta_meta_prompt_detailed_results,\n",
    "                        gpt3_5_tot_prompt_1_results,\n",
    "                        gpt3_5_tot_detailed_prompt_1_results,\n",
    "                        gpt3_5_tot_prompt_2_results,\n",
    "                        gpt3_5_tot_detailed_prompt_2_results,\n",
    "                        gpt3_5_tot_prompt_3_results,\n",
    "                        gpt3_5_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt3_5_zs_baseline_system_prompt_scores + gpt3_5_zs_baseline_system_prompt_detailed_scores + gpt3_5_zs_role_system_prompt_scores + gpt3_5_zs_role_system_prompt_detailed_scores + gpt3_5_zs_role_best_system_prompt_scores + gpt3_5_zs_role_best_system_prompt_detailed_scores + gpt3_5_cot_human_prompt_scores + gpt3_5_cot_human_prompt_detailed_scores + gpt3_5_cot_ape_prompt_scores + gpt3_5_cot_ape_prompt_detailed_scores + gpt3_5_meta_system_prompt_scores + gpt3_5_meta_system_prompt_detailed_scores + gpt3_5_meta_meta_prompt_scores + gpt3_5_meta_meta_prompt_detailed_scores + gpt3_5_tot_prompt_1_scores + gpt3_5_tot_detailed_prompt_1_scores + gpt3_5_tot_prompt_2_scores + gpt3_5_tot_detailed_prompt_2_scores + gpt3_5_tot_prompt_3_scores + gpt3_5_tot_detailed_prompt_3_scores)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Totale average zero-shot baseline average\n",
      "Succes rate: 0.36666666666666664\n",
      "Avg score: 40.372727272727275\n",
      "Totale average zero-shot baseline system prompt\n",
      "Succes rate: 0.25333333333333335\n",
      "Avg score: 37.0\n",
      "Totale average zero-shot baseline system prompt detailed\n",
      "Succes rate: 0.48\n",
      "Avg score: 42.15277777777778\n",
      "Totale average zero-shot role system average\n",
      "Succes rate: 0.37333333333333335\n",
      "Avg score: 40.88392857142857\n",
      "Totale average zero-shot role system prompt\n",
      "Succes rate: 0.20666666666666667\n",
      "Avg score: 37.96774193548387\n",
      "Totale average zero-shot role system prompt detailed\n",
      "Succes rate: 0.54\n",
      "Avg score: 42.0\n",
      "Totale average zero-shot role best system average\n",
      "Succes rate: 0.37333333333333335\n",
      "Avg score: 41.25\n",
      "Totale average zero-shot role best system prompt\n",
      "Succes rate: 0.23333333333333334\n",
      "Avg score: 38.65714285714286\n",
      "Totale average zero-shot role best system prompt detailed\n",
      "Succes rate: 0.5133333333333333\n",
      "Avg score: 42.42857142857143\n",
      "Totale average cot human average\n",
      "Succes rate: 0.4266666666666667\n",
      "Avg score: 40.046875\n",
      "Totale average cot human prompt\n",
      "Succes rate: 0.38666666666666666\n",
      "Avg score: 37.810344827586206\n",
      "Totale average cot human prompt detailed\n",
      "Succes rate: 0.4666666666666667\n",
      "Avg score: 41.9\n",
      "Totale average cot ape average\n",
      "Succes rate: 0.4166666666666667\n",
      "Avg score: 39.552\n",
      "Totale average cot ape prompt\n",
      "Succes rate: 0.38666666666666666\n",
      "Avg score: 37.51724137931034\n",
      "Totale average cot ape prompt detailed\n",
      "Succes rate: 0.44666666666666666\n",
      "Avg score: 41.3134328358209\n",
      "Totale average Meta system average\n",
      "Succes rate: 0.39666666666666667\n",
      "Avg score: 40.87394957983193\n",
      "Totale average Meta system prompt\n",
      "Succes rate: 0.26\n",
      "Avg score: 39.35897435897436\n",
      "Totale average Meta system prompt detailed\n",
      "Succes rate: 0.5333333333333333\n",
      "Avg score: 41.6125\n",
      "Totale average Meta meta average\n",
      "Succes rate: 0.09666666666666666\n",
      "Avg score: 51.3448275862069\n",
      "Totale average Meta meta prompt\n",
      "Succes rate: 0.04666666666666667\n",
      "Avg score: 51.42857142857143\n",
      "Totale average Meta meta prompt detailed\n",
      "Succes rate: 0.14666666666666667\n",
      "Avg score: 51.31818181818182\n",
      "Totale average tot prompt 1 average\n",
      "Succes rate: 0.3233333333333333\n",
      "Avg score: 40.16494845360825\n",
      "Totale average tot prompt 1\n",
      "Succes rate: 0.2866666666666667\n",
      "Avg score: 37.325581395348834\n",
      "Totale average tot detailed prompt 1\n",
      "Succes rate: 0.36\n",
      "Avg score: 42.425925925925924\n",
      "Totale average tot prompt 2 average\n",
      "Succes rate: 0.23666666666666666\n",
      "Avg score: 40.50704225352113\n",
      "Totale average tot prompt 2\n",
      "Succes rate: 0.16\n",
      "Avg score: 37.708333333333336\n",
      "Totale average tot detailed prompt 2\n",
      "Succes rate: 0.31333333333333335\n",
      "Avg score: 41.93617021276596\n",
      "Totale average tot prompt 3 average\n",
      "Succes rate: 0.19333333333333333\n",
      "Avg score: 40.189655172413794\n",
      "Totale average tot prompt 3\n",
      "Succes rate: 0.18\n",
      "Avg score: 38.074074074074076\n",
      "Totale average tot detailed prompt 3\n",
      "Succes rate: 0.20666666666666667\n",
      "Avg score: 42.03225806451613\n",
      "Totale detailed average\n",
      "Succes rate: 0.40066666666666667\n",
      "Avg score: 42.30948419301165\n",
      "Totale not detailed average\n",
      "Succes rate: 0.24\n",
      "Avg score: 38.16111111111111\n",
      "Totale average zero-shot average\n",
      "Succes rate: 0.3711111111111111\n",
      "Avg score: 40.83832335329341\n",
      "Totale average cot average\n",
      "Succes rate: 0.4216666666666667\n",
      "Avg score: 39.80237154150198\n",
      "Totale average Meta average\n",
      "Succes rate: 0.24666666666666667\n",
      "Avg score: 42.92567567567568\n",
      "Totale average tot average\n",
      "Succes rate: 0.2511111111111111\n",
      "Avg score: 40.2787610619469\n",
      "Totale totale average\n",
      "Succes rate: 0.32033333333333336\n",
      "Avg score: 40.755463059313215\n"
     ]
    }
   ],
   "source": [
    "# Totale average\n",
    "print(\"Totale average zero-shot baseline average\")\n",
    "calculate_success_rate([gpt3_5_zs_baseline_system_prompt_results, gpt4_zs_baseline_system_prompt_results, gpt4o_zs_baseline_system_prompt_results, gpt3_5_zs_baseline_system_prompt_detailed_results, gpt4_zs_baseline_system_prompt_detailed_results, gpt4o_zs_baseline_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_zs_baseline_system_prompt_scores + gpt4_zs_baseline_system_prompt_scores + gpt4o_zs_baseline_system_prompt_scores + gpt3_5_zs_baseline_system_prompt_detailed_scores + gpt4_zs_baseline_system_prompt_detailed_scores + gpt4o_zs_baseline_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"Totale average zero-shot baseline system prompt\")\n",
    "calculate_success_rate([gpt3_5_zs_baseline_system_prompt_results, gpt4_zs_baseline_system_prompt_results, gpt4o_zs_baseline_system_prompt_results])\n",
    "calculate_average_score(gpt3_5_zs_baseline_system_prompt_scores + gpt4_zs_baseline_system_prompt_scores + gpt4o_zs_baseline_system_prompt_scores)\n",
    "\n",
    "print(\"Totale average zero-shot baseline system prompt detailed\")\n",
    "calculate_success_rate([gpt3_5_zs_baseline_system_prompt_detailed_results, gpt4_zs_baseline_system_prompt_detailed_results, gpt4o_zs_baseline_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_zs_baseline_system_prompt_detailed_scores + gpt4_zs_baseline_system_prompt_detailed_scores + gpt4o_zs_baseline_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"Totale average zero-shot role system average\")\n",
    "calculate_success_rate([gpt3_5_zs_role_system_prompt_results, gpt4_zs_role_system_prompt_results, gpt4o_zs_role_system_prompt_results, gpt3_5_zs_role_system_prompt_detailed_results, gpt4_zs_role_system_prompt_detailed_results, gpt4o_zs_role_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_zs_role_system_prompt_scores + gpt4_zs_role_system_prompt_scores + gpt4o_zs_role_system_prompt_scores + gpt3_5_zs_role_system_prompt_detailed_scores + gpt4_zs_role_system_prompt_detailed_scores + gpt4o_zs_role_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"Totale average zero-shot role system prompt\")\n",
    "calculate_success_rate([gpt3_5_zs_role_system_prompt_results, gpt4_zs_role_system_prompt_results, gpt4o_zs_role_system_prompt_results])\n",
    "calculate_average_score(gpt3_5_zs_role_system_prompt_scores + gpt4_zs_role_system_prompt_scores + gpt4o_zs_role_system_prompt_scores)\n",
    "\n",
    "print(\"Totale average zero-shot role system prompt detailed\")\n",
    "calculate_success_rate([gpt3_5_zs_role_system_prompt_detailed_results, gpt4_zs_role_system_prompt_detailed_results, gpt4o_zs_role_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_zs_role_system_prompt_detailed_scores + gpt4_zs_role_system_prompt_detailed_scores + gpt4o_zs_role_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"Totale average zero-shot role best system average\")\n",
    "calculate_success_rate([gpt3_5_zs_role_best_system_prompt_results, gpt4_zs_role_best_system_prompt_results, gpt4o_zs_role_best_system_prompt_results, gpt3_5_zs_role_best_system_prompt_detailed_results, gpt4_zs_role_best_system_prompt_detailed_results, gpt4o_zs_role_best_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_zs_role_best_system_prompt_scores + gpt4_zs_role_best_system_prompt_scores + gpt4o_zs_role_best_system_prompt_scores + gpt3_5_zs_role_best_system_prompt_detailed_scores + gpt4_zs_role_best_system_prompt_detailed_scores + gpt4o_zs_role_best_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"Totale average zero-shot role best system prompt\")\n",
    "calculate_success_rate([gpt3_5_zs_role_best_system_prompt_results, gpt4_zs_role_best_system_prompt_results, gpt4o_zs_role_best_system_prompt_results])\n",
    "calculate_average_score(gpt3_5_zs_role_best_system_prompt_scores + gpt4_zs_role_best_system_prompt_scores + gpt4o_zs_role_best_system_prompt_scores)\n",
    "\n",
    "print(\"Totale average zero-shot role best system prompt detailed\")\n",
    "calculate_success_rate([gpt3_5_zs_role_best_system_prompt_detailed_results, gpt4_zs_role_best_system_prompt_detailed_results, gpt4o_zs_role_best_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_zs_role_best_system_prompt_detailed_scores + gpt4_zs_role_best_system_prompt_detailed_scores + gpt4o_zs_role_best_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"Totale average cot human average\")\n",
    "calculate_success_rate([gpt3_5_cot_human_prompt_results, gpt4_cot_human_prompt_results, gpt4o_cot_human_prompt_results, gpt3_5_cot_human_prompt_detailed_results, gpt4_cot_human_prompt_detailed_results, gpt4o_cot_human_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_cot_human_prompt_scores + gpt4_cot_human_prompt_scores + gpt4o_cot_human_prompt_scores + gpt3_5_cot_human_prompt_detailed_scores + gpt4_cot_human_prompt_detailed_scores + gpt4o_cot_human_prompt_detailed_scores)\n",
    "\n",
    "print(\"Totale average cot human prompt\")\n",
    "calculate_success_rate([gpt3_5_cot_human_prompt_results, gpt4_cot_human_prompt_results, gpt4o_cot_human_prompt_results])\n",
    "calculate_average_score(gpt3_5_cot_human_prompt_scores + gpt4_cot_human_prompt_scores + gpt4o_cot_human_prompt_scores)\n",
    "\n",
    "print(\"Totale average cot human prompt detailed\")\n",
    "calculate_success_rate([gpt3_5_cot_human_prompt_detailed_results, gpt4_cot_human_prompt_detailed_results, gpt4o_cot_human_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_cot_human_prompt_detailed_scores + gpt4_cot_human_prompt_detailed_scores + gpt4o_cot_human_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"Totale average cot ape average\")\n",
    "calculate_success_rate([gpt3_5_cot_ape_prompt_results, gpt4_cot_ape_prompt_results, gpt4o_cot_ape_prompt_results, gpt3_5_cot_ape_prompt_detailed_results, gpt4_cot_ape_prompt_detailed_results, gpt4o_cot_ape_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_cot_ape_prompt_scores + gpt4_cot_ape_prompt_scores + gpt4o_cot_ape_prompt_scores + gpt3_5_cot_ape_prompt_detailed_scores + gpt4_cot_ape_prompt_detailed_scores + gpt4o_cot_ape_prompt_detailed_scores)\n",
    "\n",
    "print(\"Totale average cot ape prompt\")\n",
    "calculate_success_rate([gpt3_5_cot_ape_prompt_results, gpt4_cot_ape_prompt_results, gpt4o_cot_ape_prompt_results])\n",
    "calculate_average_score(gpt3_5_cot_ape_prompt_scores + gpt4_cot_ape_prompt_scores + gpt4o_cot_ape_prompt_scores)\n",
    "\n",
    "print(\"Totale average cot ape prompt detailed\")\n",
    "calculate_success_rate([gpt3_5_cot_ape_prompt_detailed_results, gpt4_cot_ape_prompt_detailed_results, gpt4o_cot_ape_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_cot_ape_prompt_detailed_scores + gpt4_cot_ape_prompt_detailed_scores + gpt4o_cot_ape_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"Totale average Meta system average\")\n",
    "calculate_success_rate([gpt3_5_meta_system_prompt_results, gpt4_meta_system_prompt_results, gpt4o_meta_system_prompt_results, gpt3_5_meta_system_prompt_detailed_results, gpt4_meta_system_prompt_detailed_results, gpt4o_meta_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_meta_system_prompt_scores + gpt4_meta_system_prompt_scores + gpt4o_meta_system_prompt_scores + gpt3_5_meta_system_prompt_detailed_scores + gpt4_meta_system_prompt_detailed_scores + gpt4o_meta_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"Totale average Meta system prompt\")\n",
    "calculate_success_rate([gpt3_5_meta_system_prompt_results, gpt4_meta_system_prompt_results, gpt4o_meta_system_prompt_results])\n",
    "calculate_average_score(gpt3_5_meta_system_prompt_scores + gpt4_meta_system_prompt_scores + gpt4o_meta_system_prompt_scores)\n",
    "\n",
    "print(\"Totale average Meta system prompt detailed\")\n",
    "calculate_success_rate([gpt3_5_meta_system_prompt_detailed_results, gpt4_meta_system_prompt_detailed_results, gpt4o_meta_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_meta_system_prompt_detailed_scores + gpt4_meta_system_prompt_detailed_scores + gpt4o_meta_system_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"Totale average Meta meta average\")\n",
    "calculate_success_rate([gpt3_5_meta_meta_prompt_results, gpt4_meta_meta_prompt_results, gpt4o_meta_meta_prompt_results, gpt3_5_meta_meta_prompt_detailed_results, gpt4_meta_meta_prompt_detailed_results, gpt4o_meta_meta_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_meta_meta_prompt_scores + gpt4_meta_meta_prompt_scores + gpt4o_meta_meta_prompt_scores + gpt3_5_meta_meta_prompt_detailed_scores + gpt4_meta_meta_prompt_detailed_scores + gpt4o_meta_meta_prompt_detailed_scores)\n",
    "\n",
    "print(\"Totale average Meta meta prompt\")\n",
    "calculate_success_rate([gpt3_5_meta_meta_prompt_results, gpt4_meta_meta_prompt_results, gpt4o_meta_meta_prompt_results])\n",
    "calculate_average_score(gpt3_5_meta_meta_prompt_scores + gpt4_meta_meta_prompt_scores + gpt4o_meta_meta_prompt_scores)\n",
    "\n",
    "print(\"Totale average Meta meta prompt detailed\")\n",
    "calculate_success_rate([gpt3_5_meta_meta_prompt_detailed_results, gpt4_meta_meta_prompt_detailed_results, gpt4o_meta_meta_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_meta_meta_prompt_detailed_scores + gpt4_meta_meta_prompt_detailed_scores + gpt4o_meta_meta_prompt_detailed_scores)\n",
    "\n",
    "####\n",
    "print(\"Totale average tot prompt 1 average\")\n",
    "calculate_success_rate([gpt3_5_tot_prompt_1_results, gpt4_tot_prompt_1_results, gpt4o_tot_prompt_1_results, gpt3_5_tot_detailed_prompt_1_results, gpt4_tot_detailed_prompt_1_results, gpt4o_tot_detailed_prompt_1_results])\n",
    "calculate_average_score(gpt3_5_tot_prompt_1_scores + gpt4_tot_prompt_1_scores + gpt4o_tot_prompt_1_scores + gpt3_5_tot_detailed_prompt_1_scores + gpt4_tot_detailed_prompt_1_scores + gpt4o_tot_detailed_prompt_1_scores)\n",
    "\n",
    "print(\"Totale average tot prompt 1\")\n",
    "calculate_success_rate([gpt3_5_tot_prompt_1_results, gpt4_tot_prompt_1_results, gpt4o_tot_prompt_1_results])\n",
    "calculate_average_score(gpt3_5_tot_prompt_1_scores + gpt4_tot_prompt_1_scores + gpt4o_tot_prompt_1_scores)\n",
    "\n",
    "print(\"Totale average tot detailed prompt 1\")\n",
    "calculate_success_rate([gpt3_5_tot_detailed_prompt_1_results, gpt4_tot_detailed_prompt_1_results, gpt4o_tot_detailed_prompt_1_results])\n",
    "calculate_average_score(gpt3_5_tot_detailed_prompt_1_scores + gpt4_tot_detailed_prompt_1_scores + gpt4o_tot_detailed_prompt_1_scores)\n",
    "\n",
    "####\n",
    "print(\"Totale average tot prompt 2 average\")\n",
    "calculate_success_rate([gpt3_5_tot_prompt_2_results, gpt4_tot_prompt_2_results, gpt4o_tot_prompt_2_results, gpt3_5_tot_detailed_prompt_2_results, gpt4_tot_detailed_prompt_2_results, gpt4o_tot_detailed_prompt_2_results])\n",
    "calculate_average_score(gpt3_5_tot_prompt_2_scores + gpt4_tot_prompt_2_scores + gpt4o_tot_prompt_2_scores + gpt3_5_tot_detailed_prompt_2_scores + gpt4_tot_detailed_prompt_2_scores + gpt4o_tot_detailed_prompt_2_scores)\n",
    "\n",
    "print(\"Totale average tot prompt 2\")\n",
    "calculate_success_rate([gpt3_5_tot_prompt_2_results, gpt4_tot_prompt_2_results, gpt4o_tot_prompt_2_results])\n",
    "calculate_average_score(gpt3_5_tot_prompt_2_scores + gpt4_tot_prompt_2_scores + gpt4o_tot_prompt_2_scores)\n",
    "\n",
    "print(\"Totale average tot detailed prompt 2\")\n",
    "calculate_success_rate([gpt3_5_tot_detailed_prompt_2_results, gpt4_tot_detailed_prompt_2_results, gpt4o_tot_detailed_prompt_2_results])\n",
    "calculate_average_score(gpt3_5_tot_detailed_prompt_2_scores + gpt4_tot_detailed_prompt_2_scores + gpt4o_tot_detailed_prompt_2_scores)\n",
    "\n",
    "####\n",
    "print(\"Totale average tot prompt 3 average\")\n",
    "calculate_success_rate([gpt3_5_tot_prompt_3_results, gpt4_tot_prompt_3_results, gpt4o_tot_prompt_3_results, gpt3_5_tot_detailed_prompt_3_results, gpt4_tot_detailed_prompt_3_results, gpt4o_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt3_5_tot_prompt_3_scores + gpt4_tot_prompt_3_scores + gpt4o_tot_prompt_3_scores + gpt3_5_tot_detailed_prompt_3_scores + gpt4_tot_detailed_prompt_3_scores + gpt4o_tot_detailed_prompt_3_scores)\n",
    "\n",
    "print(\"Totale average tot prompt 3\")\n",
    "calculate_success_rate([gpt3_5_tot_prompt_3_results, gpt4_tot_prompt_3_results, gpt4o_tot_prompt_3_results])\n",
    "calculate_average_score(gpt3_5_tot_prompt_3_scores + gpt4_tot_prompt_3_scores + gpt4o_tot_prompt_3_scores)\n",
    "\n",
    "print(\"Totale average tot detailed prompt 3\")\n",
    "calculate_success_rate([gpt3_5_tot_detailed_prompt_3_results, gpt4_tot_detailed_prompt_3_results, gpt4o_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt3_5_tot_detailed_prompt_3_scores + gpt4_tot_detailed_prompt_3_scores + gpt4o_tot_detailed_prompt_3_scores)\n",
    "\n",
    "####\n",
    "# Totale detailed average\n",
    "print(\"Totale detailed average\")\n",
    "calculate_success_rate([gpt3_5_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt4_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt4o_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt3_5_zs_role_system_prompt_detailed_results,\n",
    "                        gpt4_zs_role_system_prompt_detailed_results,\n",
    "                        gpt4o_zs_role_system_prompt_detailed_results,\n",
    "                        gpt3_5_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt4_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt4o_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt3_5_cot_human_prompt_detailed_results,\n",
    "                        gpt4_cot_human_prompt_detailed_results,\n",
    "                        gpt4o_cot_human_prompt_detailed_results,\n",
    "                        gpt3_5_cot_ape_prompt_detailed_results,\n",
    "                        gpt4_cot_ape_prompt_detailed_results,\n",
    "                        gpt4o_cot_ape_prompt_detailed_results,\n",
    "                        gpt3_5_meta_system_prompt_detailed_results,\n",
    "                        gpt4_meta_system_prompt_detailed_results,\n",
    "                        gpt4o_meta_system_prompt_detailed_results,\n",
    "                        gpt3_5_meta_meta_prompt_detailed_results,\n",
    "                        gpt4_meta_meta_prompt_detailed_results,\n",
    "                        gpt4o_meta_meta_prompt_detailed_results,\n",
    "                        gpt3_5_tot_detailed_prompt_1_results,\n",
    "                        gpt4_tot_detailed_prompt_1_results,\n",
    "                        gpt4o_tot_detailed_prompt_1_results,\n",
    "                        gpt3_5_tot_detailed_prompt_2_results,\n",
    "                        gpt4_tot_detailed_prompt_2_results,\n",
    "                        gpt4o_tot_detailed_prompt_2_results,\n",
    "                        gpt3_5_tot_detailed_prompt_3_results,\n",
    "                        gpt4_tot_detailed_prompt_3_results,\n",
    "                        gpt4o_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt3_5_zs_baseline_system_prompt_detailed_scores + gpt4_zs_baseline_system_prompt_detailed_scores + gpt4o_zs_baseline_system_prompt_detailed_scores + gpt3_5_zs_role_system_prompt_detailed_scores + gpt4_zs_role_system_prompt_detailed_scores + gpt4o_zs_role_system_prompt_detailed_scores + gpt3_5_zs_role_best_system_prompt_detailed_scores + gpt4_zs_role_best_system_prompt_detailed_scores + gpt4o_zs_role_best_system_prompt_detailed_scores + gpt3_5_cot_human_prompt_detailed_scores + gpt4_cot_human_prompt_detailed_scores + gpt4o_cot_human_prompt_detailed_scores + gpt3_5_cot_ape_prompt_detailed_scores + gpt4_cot_ape_prompt_detailed_scores + gpt4o_cot_ape_prompt_detailed_scores + gpt3_5_meta_system_prompt_detailed_scores + gpt4_meta_system_prompt_detailed_scores + gpt4o_meta_system_prompt_detailed_scores + gpt3_5_meta_meta_prompt_detailed_scores + gpt4_meta_meta_prompt_detailed_scores + gpt4o_meta_meta_prompt_detailed_scores + gpt3_5_tot_detailed_prompt_1_scores + gpt4_tot_detailed_prompt_1_scores + gpt4o_tot_detailed_prompt_1_scores + gpt3_5_tot_detailed_prompt_2_scores + gpt4_tot_detailed_prompt_2_scores + gpt4o_tot_detailed_prompt_2_scores + gpt3_5_tot_detailed_prompt_3_scores + gpt4_tot_detailed_prompt_3_scores + gpt4o_tot_detailed_prompt_3_scores)\n",
    "\n",
    "# Totale not detailed average\n",
    "print(\"Totale not detailed average\")\n",
    "calculate_success_rate([gpt3_5_zs_baseline_system_prompt_results,\n",
    "                        gpt4_zs_baseline_system_prompt_results,\n",
    "                        gpt4o_zs_baseline_system_prompt_results,\n",
    "                        gpt3_5_zs_role_system_prompt_results,\n",
    "                        gpt4_zs_role_system_prompt_results,\n",
    "                        gpt4o_zs_role_system_prompt_results,\n",
    "                        gpt3_5_zs_role_best_system_prompt_results,\n",
    "                        gpt4_zs_role_best_system_prompt_results,\n",
    "                        gpt4o_zs_role_best_system_prompt_results,\n",
    "                        gpt3_5_cot_human_prompt_results,\n",
    "                        gpt4_cot_human_prompt_results,\n",
    "                        gpt4o_cot_human_prompt_results,\n",
    "                        gpt3_5_cot_ape_prompt_results,\n",
    "                        gpt4_cot_ape_prompt_results,\n",
    "                        gpt4o_cot_ape_prompt_results,\n",
    "                        gpt3_5_meta_system_prompt_results,\n",
    "                        gpt4_meta_system_prompt_results,\n",
    "                        gpt4o_meta_system_prompt_results,\n",
    "                        gpt3_5_meta_meta_prompt_results,\n",
    "                        gpt4_meta_meta_prompt_results,\n",
    "                        gpt4o_meta_meta_prompt_results,\n",
    "                        gpt3_5_tot_prompt_1_results,\n",
    "                        gpt4_tot_prompt_1_results,\n",
    "                        gpt4o_tot_prompt_1_results,\n",
    "                        gpt3_5_tot_prompt_2_results,\n",
    "                        gpt4_tot_prompt_2_results,\n",
    "                        gpt4o_tot_prompt_2_results,\n",
    "                        gpt3_5_tot_prompt_3_results,\n",
    "                        gpt4_tot_prompt_3_results,\n",
    "                        gpt4o_tot_prompt_3_results])\n",
    "calculate_average_score(gpt3_5_zs_baseline_system_prompt_scores + gpt4_zs_baseline_system_prompt_scores + gpt4o_zs_baseline_system_prompt_scores + gpt3_5_zs_role_system_prompt_scores + gpt4_zs_role_system_prompt_scores + gpt4o_zs_role_system_prompt_scores + gpt3_5_zs_role_best_system_prompt_scores + gpt4_zs_role_best_system_prompt_scores + gpt4o_zs_role_best_system_prompt_scores + gpt3_5_cot_human_prompt_scores + gpt4_cot_human_prompt_scores + gpt4o_cot_human_prompt_scores + gpt3_5_cot_ape_prompt_scores + gpt4_cot_ape_prompt_scores + gpt4o_cot_ape_prompt_scores + gpt3_5_meta_system_prompt_scores + gpt4_meta_system_prompt_scores + gpt4o_meta_system_prompt_scores + gpt3_5_meta_meta_prompt_scores + gpt4_meta_meta_prompt_scores + gpt4o_meta_meta_prompt_scores + gpt3_5_tot_prompt_1_scores + gpt4_tot_prompt_1_scores + gpt4o_tot_prompt_1_scores + gpt3_5_tot_prompt_2_scores + gpt4_tot_prompt_2_scores + gpt4o_tot_prompt_2_scores + gpt3_5_tot_prompt_3_scores + gpt4_tot_prompt_3_scores + gpt4o_tot_prompt_3_scores)\n",
    "\n",
    "####\n",
    "# Totale average zero shot\n",
    "print(\"Totale average zero-shot average\")\n",
    "calculate_success_rate([gpt3_5_zs_baseline_system_prompt_results,\n",
    "                        gpt4_zs_baseline_system_prompt_results,\n",
    "                        gpt4o_zs_baseline_system_prompt_results,\n",
    "                        gpt3_5_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt4_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt4o_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt3_5_zs_role_system_prompt_results,\n",
    "                        gpt4_zs_role_system_prompt_results,\n",
    "                        gpt4o_zs_role_system_prompt_results,\n",
    "                        gpt3_5_zs_role_system_prompt_detailed_results,\n",
    "                        gpt4_zs_role_system_prompt_detailed_results,\n",
    "                        gpt4o_zs_role_system_prompt_detailed_results,\n",
    "                        gpt3_5_zs_role_best_system_prompt_results,\n",
    "                        gpt4_zs_role_best_system_prompt_results,\n",
    "                        gpt4o_zs_role_best_system_prompt_results,\n",
    "                        gpt3_5_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt4_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt4o_zs_role_best_system_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_zs_baseline_system_prompt_scores + gpt4_zs_baseline_system_prompt_scores + gpt4o_zs_baseline_system_prompt_scores + gpt3_5_zs_baseline_system_prompt_detailed_scores + gpt4_zs_baseline_system_prompt_detailed_scores + gpt4o_zs_baseline_system_prompt_detailed_scores + gpt3_5_zs_role_system_prompt_scores + gpt4_zs_role_system_prompt_scores + gpt4o_zs_role_system_prompt_scores + gpt3_5_zs_role_system_prompt_detailed_scores + gpt4_zs_role_system_prompt_detailed_scores + gpt4o_zs_role_system_prompt_detailed_scores + gpt3_5_zs_role_best_system_prompt_scores + gpt4_zs_role_best_system_prompt_scores + gpt4o_zs_role_best_system_prompt_scores + gpt3_5_zs_role_best_system_prompt_detailed_scores + gpt4_zs_role_best_system_prompt_detailed_scores + gpt4o_zs_role_best_system_prompt_detailed_scores)\n",
    "\n",
    "print(\"Totale average cot average\")\n",
    "calculate_success_rate([gpt3_5_cot_human_prompt_results,\n",
    "                        gpt4_cot_human_prompt_results,\n",
    "                        gpt4o_cot_human_prompt_results,\n",
    "                        gpt3_5_cot_human_prompt_detailed_results,\n",
    "                        gpt4_cot_human_prompt_detailed_results,\n",
    "                        gpt4o_cot_human_prompt_detailed_results,\n",
    "                        gpt3_5_cot_ape_prompt_results,\n",
    "                        gpt4_cot_ape_prompt_results,\n",
    "                        gpt4o_cot_ape_prompt_results,\n",
    "                        gpt3_5_cot_ape_prompt_detailed_results,\n",
    "                        gpt4_cot_ape_prompt_detailed_results,\n",
    "                        gpt4o_cot_ape_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_cot_human_prompt_scores + gpt4_cot_human_prompt_scores + gpt4o_cot_human_prompt_scores + gpt3_5_cot_human_prompt_detailed_scores + gpt4_cot_human_prompt_detailed_scores + gpt4o_cot_human_prompt_detailed_scores + gpt3_5_cot_ape_prompt_scores + gpt4_cot_ape_prompt_scores + gpt4o_cot_ape_prompt_scores + gpt3_5_cot_ape_prompt_detailed_scores + gpt4_cot_ape_prompt_detailed_scores + gpt4o_cot_ape_prompt_detailed_scores)\n",
    "\n",
    "print(\"Totale average Meta average\")\n",
    "calculate_success_rate([gpt3_5_meta_system_prompt_results,\n",
    "                        gpt4_meta_system_prompt_results,\n",
    "                        gpt4o_meta_system_prompt_results,\n",
    "                        gpt3_5_meta_system_prompt_detailed_results,\n",
    "                        gpt4_meta_system_prompt_detailed_results,\n",
    "                        gpt4o_meta_system_prompt_detailed_results,\n",
    "                        gpt3_5_meta_meta_prompt_results,\n",
    "                        gpt4_meta_meta_prompt_results,\n",
    "                        gpt4o_meta_meta_prompt_results,\n",
    "                        gpt3_5_meta_meta_prompt_detailed_results,\n",
    "                        gpt4_meta_meta_prompt_detailed_results,\n",
    "                        gpt4o_meta_meta_prompt_detailed_results])\n",
    "calculate_average_score(gpt3_5_meta_system_prompt_scores + gpt4_meta_system_prompt_scores + gpt4o_meta_system_prompt_scores + gpt3_5_meta_system_prompt_detailed_scores + gpt4_meta_system_prompt_detailed_scores + gpt4o_meta_system_prompt_detailed_scores + gpt3_5_meta_meta_prompt_scores + gpt4_meta_meta_prompt_scores + gpt4o_meta_meta_prompt_scores + gpt3_5_meta_meta_prompt_detailed_scores + gpt4_meta_meta_prompt_detailed_scores + gpt4o_meta_meta_prompt_detailed_scores)\n",
    "\n",
    "print(\"Totale average tot average\")\n",
    "calculate_success_rate([gpt3_5_tot_prompt_1_results,\n",
    "                        gpt4_tot_prompt_1_results,\n",
    "                        gpt4o_tot_prompt_1_results,\n",
    "                        gpt3_5_tot_detailed_prompt_1_results,\n",
    "                        gpt4_tot_detailed_prompt_1_results,\n",
    "                        gpt4o_tot_detailed_prompt_1_results,\n",
    "                        gpt3_5_tot_prompt_2_results,\n",
    "                        gpt4_tot_prompt_2_results,\n",
    "                        gpt4o_tot_prompt_2_results,\n",
    "                        gpt3_5_tot_detailed_prompt_2_results,\n",
    "                        gpt4_tot_detailed_prompt_2_results,\n",
    "                        gpt4o_tot_detailed_prompt_2_results,\n",
    "                        gpt3_5_tot_prompt_3_results,\n",
    "                        gpt4_tot_prompt_3_results,\n",
    "                        gpt4o_tot_prompt_3_results,\n",
    "                        gpt3_5_tot_detailed_prompt_3_results,\n",
    "                        gpt4_tot_detailed_prompt_3_results,\n",
    "                        gpt4o_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt3_5_tot_prompt_1_scores + gpt4_tot_prompt_1_scores + gpt4o_tot_prompt_1_scores + gpt3_5_tot_detailed_prompt_1_scores + gpt4_tot_detailed_prompt_1_scores + gpt4o_tot_detailed_prompt_1_scores + gpt3_5_tot_prompt_2_scores + gpt4_tot_prompt_2_scores + gpt4o_tot_prompt_2_scores + gpt3_5_tot_detailed_prompt_2_scores + gpt4_tot_detailed_prompt_2_scores + gpt4o_tot_detailed_prompt_2_scores + gpt3_5_tot_prompt_3_scores + gpt4_tot_prompt_3_scores + gpt4o_tot_prompt_3_scores + gpt3_5_tot_detailed_prompt_3_scores + gpt4_tot_detailed_prompt_3_scores + gpt4o_tot_detailed_prompt_3_scores)\n",
    "\n",
    "# Totale totale average\n",
    "print(\"Totale totale average\")\n",
    "calculate_success_rate([gpt3_5_zs_baseline_system_prompt_results,\n",
    "                        gpt4_zs_baseline_system_prompt_results,\n",
    "                        gpt4o_zs_baseline_system_prompt_results,\n",
    "                        gpt3_5_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt4_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt4o_zs_baseline_system_prompt_detailed_results,\n",
    "                        gpt3_5_zs_role_system_prompt_results,\n",
    "                        gpt4_zs_role_system_prompt_results,\n",
    "                        gpt4o_zs_role_system_prompt_results,\n",
    "                        gpt3_5_zs_role_system_prompt_detailed_results,\n",
    "                        gpt4_zs_role_system_prompt_detailed_results,\n",
    "                        gpt4o_zs_role_system_prompt_detailed_results,\n",
    "                        gpt3_5_zs_role_best_system_prompt_results,\n",
    "                        gpt4_zs_role_best_system_prompt_results,\n",
    "                        gpt4o_zs_role_best_system_prompt_results,\n",
    "                        gpt3_5_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt4_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt4o_zs_role_best_system_prompt_detailed_results,\n",
    "                        gpt3_5_cot_human_prompt_results,\n",
    "                        gpt4_cot_human_prompt_results,\n",
    "                        gpt4o_cot_human_prompt_results,\n",
    "                        gpt3_5_cot_human_prompt_detailed_results,\n",
    "                        gpt4_cot_human_prompt_detailed_results,\n",
    "                        gpt4o_cot_human_prompt_detailed_results,\n",
    "                        gpt3_5_cot_ape_prompt_results,\n",
    "                        gpt4_cot_ape_prompt_results,\n",
    "                        gpt4o_cot_ape_prompt_results,\n",
    "                        gpt3_5_cot_ape_prompt_detailed_results,\n",
    "                        gpt4_cot_ape_prompt_detailed_results,\n",
    "                        gpt4o_cot_ape_prompt_detailed_results,\n",
    "                        gpt3_5_meta_system_prompt_results,\n",
    "                        gpt4_meta_system_prompt_results,\n",
    "                        gpt4o_meta_system_prompt_results,\n",
    "                        gpt3_5_meta_system_prompt_detailed_results,\n",
    "                        gpt4_meta_system_prompt_detailed_results,\n",
    "                        gpt4o_meta_system_prompt_detailed_results,\n",
    "                        gpt3_5_meta_meta_prompt_results,\n",
    "                        gpt4_meta_meta_prompt_results,\n",
    "                        gpt4o_meta_meta_prompt_results,\n",
    "                        gpt3_5_meta_meta_prompt_detailed_results,\n",
    "                        gpt4_meta_meta_prompt_detailed_results,\n",
    "                        gpt4o_meta_meta_prompt_detailed_results,\n",
    "                        gpt3_5_tot_prompt_1_results,\n",
    "                        gpt4_tot_prompt_1_results,\n",
    "                        gpt4o_tot_prompt_1_results,\n",
    "                        gpt3_5_tot_detailed_prompt_1_results,\n",
    "                        gpt4_tot_detailed_prompt_1_results,\n",
    "                        gpt4o_tot_detailed_prompt_1_results,\n",
    "                        gpt3_5_tot_prompt_2_results,\n",
    "                        gpt4_tot_prompt_2_results,\n",
    "                        gpt4o_tot_prompt_2_results,\n",
    "                        gpt3_5_tot_detailed_prompt_2_results,\n",
    "                        gpt4_tot_detailed_prompt_2_results,\n",
    "                        gpt4o_tot_detailed_prompt_2_results,\n",
    "                        gpt3_5_tot_prompt_3_results,\n",
    "                        gpt4_tot_prompt_3_results,\n",
    "                        gpt4o_tot_prompt_3_results,\n",
    "                        gpt3_5_tot_detailed_prompt_3_results,\n",
    "                        gpt4_tot_detailed_prompt_3_results,\n",
    "                        gpt4o_tot_detailed_prompt_3_results])\n",
    "calculate_average_score(gpt3_5_zs_baseline_system_prompt_scores + gpt4_zs_baseline_system_prompt_scores + gpt4o_zs_baseline_system_prompt_scores + gpt3_5_zs_baseline_system_prompt_detailed_scores + gpt4_zs_baseline_system_prompt_detailed_scores + gpt4o_zs_baseline_system_prompt_detailed_scores + gpt3_5_zs_role_system_prompt_scores + gpt4_zs_role_system_prompt_scores + gpt4o_zs_role_system_prompt_scores + gpt3_5_zs_role_system_prompt_detailed_scores + gpt4_zs_role_system_prompt_detailed_scores + gpt4o_zs_role_system_prompt_detailed_scores + gpt3_5_zs_role_best_system_prompt_scores + gpt4_zs_role_best_system_prompt_scores + gpt4o_zs_role_best_system_prompt_scores + gpt3_5_zs_role_best_system_prompt_detailed_scores + gpt4_zs_role_best_system_prompt_detailed_scores + gpt4o_zs_role_best_system_prompt_detailed_scores + gpt3_5_cot_human_prompt_scores + gpt4_cot_human_prompt_scores + gpt4o_cot_human_prompt_scores + gpt3_5_cot_human_prompt_detailed_scores + gpt4_cot_human_prompt_detailed_scores + gpt4o_cot_human_prompt_detailed_scores + gpt3_5_cot_ape_prompt_scores + gpt4_cot_ape_prompt_scores + gpt4o_cot_ape_prompt_scores + gpt3_5_cot_ape_prompt_detailed_scores + gpt4_cot_ape_prompt_detailed_scores + gpt4o_cot_ape_prompt_detailed_scores + gpt3_5_meta_system_prompt_scores + gpt4_meta_system_prompt_scores + gpt4o_meta_system_prompt_scores + gpt3_5_meta_system_prompt_detailed_scores + gpt4_meta_system_prompt_detailed_scores + gpt4o_meta_system_prompt_detailed_scores + gpt3_5_meta_meta_prompt_scores + gpt4_meta_meta_prompt_scores + gpt4o_meta_meta_prompt_scores + gpt3_5_meta_meta_prompt_detailed_scores + gpt4_meta_meta_prompt_detailed_scores + gpt4o_meta_meta_prompt_detailed_scores + gpt3_5_tot_prompt_1_scores + gpt4_tot_prompt_1_scores + gpt4o_tot_prompt_1_scores + gpt3_5_tot_detailed_prompt_1_scores + gpt4_tot_detailed_prompt_1_scores + gpt4o_tot_detailed_prompt_1_scores + gpt3_5_tot_prompt_2_scores + gpt4_tot_prompt_2_scores + gpt4o_tot_prompt_2_scores + gpt3_5_tot_detailed_prompt_2_scores + gpt4_tot_detailed_prompt_2_scores + gpt4o_tot_detailed_prompt_2_scores + gpt3_5_tot_prompt_3_scores + gpt4_tot_prompt_3_scores + gpt4o_tot_prompt_3_scores + gpt3_5_tot_detailed_prompt_3_scores + gpt4_tot_detailed_prompt_3_scores + gpt4o_tot_detailed_prompt_3_scores)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Code for fails table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_fails(input):\n",
    "    if isinstance(input, list):\n",
    "        results = aggregate_all_test_results(input)\n",
    "    else:\n",
    "        results = input\n",
    "    other_deployment_errors = results[\"deployment_failed\"] - results[\"duplicate_resources\"]\n",
    "    mysql_failed = results[\"MySQL_PVC_unbound\"] + results[\"MySQL_not_ready\"]\n",
    "    other_health_errors = results[\"Service_failed\"] + results[\"Unknown_error\"] + results[\"Wordpress_PVC_unbound\"]\n",
    "    result_string = \",\" + str(results[\"yaml_not_wrapped\"]) + \",\" + str(results[\"no_mysql_or_wordpress\"]) + \",\" + str(results[\"invalid_yaml\"]) + \",\" + str(results[\"kubeconform_failed\"]) + \",\" + str(results[\"duplicate_resources\"]) + \",\" + str(other_deployment_errors) + \",\" + str(results[\"PVC_failed\"]) + \",\" + str(results[\"Missing_resources\"]) + \",\" + str(results[\"Missing_key\"]) + \",\" + str(mysql_failed) + \",\" + str(results[\"Wordpress_not_ready\"]) + \",\" + str(other_health_errors) + \",\" + str(results[\"total_responses\"] - results[\"healthy\"])\n",
    "    if (results[\"yaml_not_wrapped\"] + results[\"no_mysql_or_wordpress\"] + results[\"invalid_yaml\"] + results[\"kubeconform_failed\"] + results[\"duplicate_resources\"] + other_deployment_errors + results[\"PVC_failed\"] + results[\"Missing_resources\"] + results[\"Missing_key\"] + mysql_failed + results[\"Wordpress_not_ready\"] + other_health_errors) != (results[\"total_responses\"] - results[\"healthy\"]):\n",
    "        raise Exception(\"Error fail count does not match total responses\")\n",
    "    return result_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT-4o,2,12,9,42,4,3,12,3,0,7,280,15,389\n",
      "Zero-Shot,0,0,0,0,0,0,1,0,0,0,31,0,32\n",
      "Role-Based,0,0,0,1,1,2,0,0,0,1,58,0,63\n",
      "Zero-Shot CoT,2,0,4,2,0,0,3,0,0,1,50,3,65\n",
      "Meta prompting,0,0,0,1,0,0,0,0,0,2,25,0,28\n",
      "Meta meta prompting,0,6,2,35,0,1,3,2,0,3,22,8,82\n",
      "ToT-style prompts,0,6,3,3,3,0,5,1,0,0,94,4,119\n",
      "GPT-4,10,61,22,53,39,7,2,66,4,9,418,2,693\n",
      "Zero-Shot,0,0,0,5,0,2,0,5,1,0,48,0,61\n",
      "Role-Based,0,0,1,2,0,1,0,8,2,0,104,0,118\n",
      "Zero-Shot CoT,0,3,2,6,0,1,0,3,0,0,95,0,110\n",
      "Meta prompting,0,0,0,2,0,1,0,1,0,0,53,0,57\n",
      "Meta meta prompting,0,6,0,13,0,0,2,16,0,9,45,0,91\n",
      "ToT-style prompts,10,52,19,25,39,2,0,33,1,0,73,2,256\n",
      "GPT-3.5,4,120,22,59,51,53,11,86,9,5,526,11,957\n",
      "Zero-Shot,0,3,1,1,2,10,1,12,0,0,67,0,97\n",
      "Role-Based,0,0,3,0,1,20,7,23,1,0,140,0,195\n",
      "Zero-Shot CoT,0,3,5,3,0,6,0,7,4,0,144,0,172\n",
      "Meta prompting,0,1,0,2,0,6,2,2,0,1,82,0,96\n",
      "Meta meta prompting,0,24,1,36,0,2,0,4,0,2,25,4,98\n",
      "ToT-style prompts,4,89,12,17,48,9,1,38,4,2,68,7,299\n",
      "Total,16,193,53,154,94,63,25,155,13,21,1224,28,2039\n",
      "Zero-Shot,0,3,1,6,2,12,2,17,1,0,146,0,190\n",
      "Role-Based,0,0,4,3,2,23,7,31,3,1,302,0,376\n",
      "Zero-Shot CoT,2,6,11,11,0,7,3,10,4,1,289,3,347\n",
      "Meta prompting,0,1,0,5,0,7,2,3,0,3,160,0,181\n",
      "Meta meta prompting,0,36,3,84,0,3,5,22,0,14,92,12,271\n",
      "ToT-style prompts,14,147,34,45,90,11,6,72,5,2,235,13,674\n",
      "Total detailed prompts,12,93,25,78,34,25,8,101,13,11,486,13,899\n",
      "Total not detailed prompts,4,100,28,76,60,38,17,54,0,10,738,15,1140\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# gpt4o\n",
    "print(\"GPT-4o\" + extract_fails([gpt4o_zs_baseline_system_prompt_results,\n",
    "                                gpt4o_zs_baseline_system_prompt_detailed_results,\n",
    "                                gpt4o_zs_role_system_prompt_results,\n",
    "                                gpt4o_zs_role_system_prompt_detailed_results,\n",
    "                                gpt4o_zs_role_best_system_prompt_results,\n",
    "                                gpt4o_zs_role_best_system_prompt_detailed_results,\n",
    "                                gpt4o_cot_human_prompt_results,\n",
    "                                gpt4o_cot_human_prompt_detailed_results,\n",
    "                                gpt4o_cot_ape_prompt_results,\n",
    "                                gpt4o_cot_ape_prompt_detailed_results,\n",
    "                                gpt4o_meta_system_prompt_results,\n",
    "                                gpt4o_meta_system_prompt_detailed_results,\n",
    "                                gpt4o_meta_meta_prompt_results,\n",
    "                                gpt4o_meta_meta_prompt_detailed_results,\n",
    "                                gpt4o_tot_prompt_1_results,\n",
    "                                gpt4o_tot_detailed_prompt_1_results,\n",
    "                                gpt4o_tot_prompt_2_results,\n",
    "                                gpt4o_tot_detailed_prompt_2_results,\n",
    "                                gpt4o_tot_prompt_3_results,\n",
    "                                gpt4o_tot_detailed_prompt_3_results]))\n",
    "# gpt4o zero-shot baseline\n",
    "print(\"Zero-Shot\" + extract_fails([gpt4o_zs_baseline_system_prompt_results,\n",
    "                                   gpt4o_zs_baseline_system_prompt_detailed_results]))\n",
    "\n",
    "# gpt4o zero-shot role-based\n",
    "print(\"Role-Based\" + extract_fails([gpt4o_zs_role_system_prompt_results,\n",
    "                                    gpt4o_zs_role_system_prompt_detailed_results,\n",
    "                                    gpt4o_zs_role_best_system_prompt_results,\n",
    "                                    gpt4o_zs_role_best_system_prompt_detailed_results]))\n",
    "\n",
    "# gpt4o cot\n",
    "print(\"Zero-Shot CoT\" + extract_fails([gpt4o_cot_human_prompt_results,\n",
    "                             gpt4o_cot_human_prompt_detailed_results,\n",
    "                             gpt4o_cot_ape_prompt_results,\n",
    "                             gpt4o_cot_ape_prompt_detailed_results]))\n",
    "\n",
    "# gpt4o meta\n",
    "print(\"Meta prompting\" + extract_fails([gpt4o_meta_system_prompt_results,\n",
    "                             gpt4o_meta_system_prompt_detailed_results]))\n",
    "\n",
    "# gpt4o meta-meta\n",
    "print(\"Meta meta prompting\" + extract_fails([gpt4o_meta_meta_prompt_results,\n",
    "                             gpt4o_meta_meta_prompt_detailed_results]))\n",
    "\n",
    "# gpt4o tot\n",
    "print(\"ToT-style prompts\" + extract_fails([gpt4o_tot_prompt_1_results,\n",
    "                             gpt4o_tot_detailed_prompt_1_results,\n",
    "                             gpt4o_tot_prompt_2_results,\n",
    "                             gpt4o_tot_detailed_prompt_2_results,\n",
    "                             gpt4o_tot_prompt_3_results,\n",
    "                             gpt4o_tot_detailed_prompt_3_results]))\n",
    "\n",
    "# gpt4\n",
    "print(\"GPT-4\" + extract_fails([gpt4_zs_baseline_system_prompt_results,\n",
    "                                gpt4_zs_baseline_system_prompt_detailed_results,\n",
    "                                gpt4_zs_role_system_prompt_results,\n",
    "                                gpt4_zs_role_system_prompt_detailed_results,\n",
    "                                gpt4_zs_role_best_system_prompt_results,\n",
    "                                gpt4_zs_role_best_system_prompt_detailed_results,\n",
    "                                gpt4_cot_human_prompt_results,\n",
    "                                gpt4_cot_human_prompt_detailed_results,\n",
    "                                gpt4_cot_ape_prompt_results,\n",
    "                                gpt4_cot_ape_prompt_detailed_results,\n",
    "                                gpt4_meta_system_prompt_results,\n",
    "                                gpt4_meta_system_prompt_detailed_results,\n",
    "                                gpt4_meta_meta_prompt_results,\n",
    "                                gpt4_meta_meta_prompt_detailed_results,\n",
    "                                gpt4_tot_prompt_1_results,\n",
    "                                gpt4_tot_detailed_prompt_1_results,\n",
    "                                gpt4_tot_prompt_2_results,\n",
    "                                gpt4_tot_detailed_prompt_2_results,\n",
    "                                gpt4_tot_prompt_3_results,\n",
    "                                gpt4_tot_detailed_prompt_3_results]))\n",
    "\n",
    "# gpt4 zero-shot baseline\n",
    "print(\"Zero-Shot\" + extract_fails([gpt4_zs_baseline_system_prompt_results,\n",
    "                                   gpt4_zs_baseline_system_prompt_detailed_results]))\n",
    "\n",
    "# gpt4 zero-shot role-based\n",
    "print(\"Role-Based\" + extract_fails([gpt4_zs_role_system_prompt_results,\n",
    "                                    gpt4_zs_role_system_prompt_detailed_results,\n",
    "                                    gpt4_zs_role_best_system_prompt_results,\n",
    "                                    gpt4_zs_role_best_system_prompt_detailed_results]))\n",
    "\n",
    "# gpt4 cot\n",
    "print(\"Zero-Shot CoT\" + extract_fails([gpt4_cot_human_prompt_results,\n",
    "                             gpt4_cot_human_prompt_detailed_results,\n",
    "                             gpt4_cot_ape_prompt_results,\n",
    "                             gpt4_cot_ape_prompt_detailed_results]))\n",
    "\n",
    "# gpt4 meta\n",
    "print(\"Meta prompting\" + extract_fails([gpt4_meta_system_prompt_results,\n",
    "                             gpt4_meta_system_prompt_detailed_results]))\n",
    "\n",
    "# gpt4 meta-meta\n",
    "print(\"Meta meta prompting\" + extract_fails([gpt4_meta_meta_prompt_results,\n",
    "                             gpt4_meta_meta_prompt_detailed_results]))\n",
    "\n",
    "# gpt4 tot\n",
    "print(\"ToT-style prompts\" + extract_fails([gpt4_tot_prompt_1_results,\n",
    "                             gpt4_tot_detailed_prompt_1_results,\n",
    "                             gpt4_tot_prompt_2_results,\n",
    "                             gpt4_tot_detailed_prompt_2_results,\n",
    "                             gpt4_tot_prompt_3_results,\n",
    "                             gpt4_tot_detailed_prompt_3_results]))\n",
    "\n",
    "# gpt3.5\n",
    "print(\"GPT-3.5\" + extract_fails([gpt3_5_zs_baseline_system_prompt_results,\n",
    "                                gpt3_5_zs_baseline_system_prompt_detailed_results,\n",
    "                                gpt3_5_zs_role_system_prompt_results,\n",
    "                                gpt3_5_zs_role_system_prompt_detailed_results,\n",
    "                                gpt3_5_zs_role_best_system_prompt_results,\n",
    "                                gpt3_5_zs_role_best_system_prompt_detailed_results,\n",
    "                                gpt3_5_cot_human_prompt_results,\n",
    "                                gpt3_5_cot_human_prompt_detailed_results,\n",
    "                                gpt3_5_cot_ape_prompt_results,\n",
    "                                gpt3_5_cot_ape_prompt_detailed_results,\n",
    "                                gpt3_5_meta_system_prompt_results,\n",
    "                                gpt3_5_meta_system_prompt_detailed_results,\n",
    "                                gpt3_5_meta_meta_prompt_results,\n",
    "                                gpt3_5_meta_meta_prompt_detailed_results,\n",
    "                                gpt3_5_tot_prompt_1_results,\n",
    "                                gpt3_5_tot_detailed_prompt_1_results,\n",
    "                                gpt3_5_tot_prompt_2_results,\n",
    "                                gpt3_5_tot_detailed_prompt_2_results,\n",
    "                                gpt3_5_tot_prompt_3_results,\n",
    "                                gpt3_5_tot_detailed_prompt_3_results]))\n",
    "\n",
    "# gpt3.5 zero-shot baseline\n",
    "print(\"Zero-Shot\" + extract_fails([gpt3_5_zs_baseline_system_prompt_results,\n",
    "                                   gpt3_5_zs_baseline_system_prompt_detailed_results]))\n",
    "\n",
    "# gpt3.5 zero-shot role-based\n",
    "print(\"Role-Based\" + extract_fails([gpt3_5_zs_role_system_prompt_results,\n",
    "                                    gpt3_5_zs_role_system_prompt_detailed_results,\n",
    "                                    gpt3_5_zs_role_best_system_prompt_results,\n",
    "                                    gpt3_5_zs_role_best_system_prompt_detailed_results]))\n",
    "\n",
    "# gpt3.5 cot\n",
    "print(\"Zero-Shot CoT\" + extract_fails([gpt3_5_cot_human_prompt_results,\n",
    "                             gpt3_5_cot_human_prompt_detailed_results,\n",
    "                             gpt3_5_cot_ape_prompt_results,\n",
    "                             gpt3_5_cot_ape_prompt_detailed_results]))\n",
    "\n",
    "# gpt3.5 meta\n",
    "print(\"Meta prompting\" + extract_fails([gpt3_5_meta_system_prompt_results,\n",
    "                             gpt3_5_meta_system_prompt_detailed_results]))\n",
    "\n",
    "# gpt3.5 meta-meta\n",
    "print(\"Meta meta prompting\" + extract_fails([gpt3_5_meta_meta_prompt_results,\n",
    "                             gpt3_5_meta_meta_prompt_detailed_results]))\n",
    "\n",
    "# gpt3.5 tot\n",
    "print(\"ToT-style prompts\" + extract_fails([gpt3_5_tot_prompt_1_results,\n",
    "                             gpt3_5_tot_detailed_prompt_1_results,\n",
    "                             gpt3_5_tot_prompt_2_results,\n",
    "                             gpt3_5_tot_detailed_prompt_2_results,\n",
    "                             gpt3_5_tot_prompt_3_results,\n",
    "                             gpt3_5_tot_detailed_prompt_3_results]))\n",
    "\n",
    "# Total\n",
    "print(\"Total\" + extract_fails([gpt4o_zs_baseline_system_prompt_results,\n",
    "                                gpt4o_zs_baseline_system_prompt_detailed_results,\n",
    "                                gpt4o_zs_role_system_prompt_results,\n",
    "                                gpt4o_zs_role_system_prompt_detailed_results,\n",
    "                                gpt4o_zs_role_best_system_prompt_results,\n",
    "                                gpt4o_zs_role_best_system_prompt_detailed_results,\n",
    "                                gpt4o_cot_human_prompt_results,\n",
    "                                gpt4o_cot_human_prompt_detailed_results,\n",
    "                                gpt4o_cot_ape_prompt_results,\n",
    "                                gpt4o_cot_ape_prompt_detailed_results,\n",
    "                                gpt4o_meta_system_prompt_results,\n",
    "                                gpt4o_meta_system_prompt_detailed_results,\n",
    "                                gpt4o_meta_meta_prompt_results,\n",
    "                                gpt4o_meta_meta_prompt_detailed_results,\n",
    "                                gpt4o_tot_prompt_1_results,\n",
    "                                gpt4o_tot_detailed_prompt_1_results,\n",
    "                                gpt4o_tot_prompt_2_results,\n",
    "                                gpt4o_tot_detailed_prompt_2_results,\n",
    "                                gpt4o_tot_prompt_3_results,\n",
    "                                gpt4o_tot_detailed_prompt_3_results,\n",
    "                                gpt4_zs_baseline_system_prompt_results,\n",
    "                                gpt4_zs_baseline_system_prompt_detailed_results,\n",
    "                                gpt4_zs_role_system_prompt_results,\n",
    "                                gpt4_zs_role_system_prompt_detailed_results,\n",
    "                                gpt4_zs_role_best_system_prompt_results,\n",
    "                                gpt4_zs_role_best_system_prompt_detailed_results,\n",
    "                                gpt4_cot_human_prompt_results,\n",
    "                                gpt4_cot_human_prompt_detailed_results,\n",
    "                                gpt4_cot_ape_prompt_results,\n",
    "                                gpt4_cot_ape_prompt_detailed_results,\n",
    "                                gpt4_meta_system_prompt_results,\n",
    "                                gpt4_meta_system_prompt_detailed_results,\n",
    "                                gpt4_meta_meta_prompt_results,\n",
    "                                gpt4_meta_meta_prompt_detailed_results,\n",
    "                                gpt4_tot_prompt_1_results,\n",
    "                                gpt4_tot_detailed_prompt_1_results,\n",
    "                                gpt4_tot_prompt_2_results,\n",
    "                                gpt4_tot_detailed_prompt_2_results,\n",
    "                                gpt4_tot_prompt_3_results,\n",
    "                                gpt4_tot_detailed_prompt_3_results,\n",
    "                                gpt3_5_zs_baseline_system_prompt_results,\n",
    "                                gpt3_5_zs_baseline_system_prompt_detailed_results,\n",
    "                                gpt3_5_zs_role_system_prompt_results,\n",
    "                                gpt3_5_zs_role_system_prompt_detailed_results,\n",
    "                                gpt3_5_zs_role_best_system_prompt_results,\n",
    "                                gpt3_5_zs_role_best_system_prompt_detailed_results,\n",
    "                                gpt3_5_cot_human_prompt_results,\n",
    "                                gpt3_5_cot_human_prompt_detailed_results,\n",
    "                                gpt3_5_cot_ape_prompt_results,\n",
    "                                gpt3_5_cot_ape_prompt_detailed_results,\n",
    "                                gpt3_5_meta_system_prompt_results,\n",
    "                                gpt3_5_meta_system_prompt_detailed_results,\n",
    "                                gpt3_5_meta_meta_prompt_results,\n",
    "                                gpt3_5_meta_meta_prompt_detailed_results,\n",
    "                                gpt3_5_tot_prompt_1_results,\n",
    "                                gpt3_5_tot_detailed_prompt_1_results,\n",
    "                                gpt3_5_tot_prompt_2_results,\n",
    "                                gpt3_5_tot_detailed_prompt_2_results,\n",
    "                                gpt3_5_tot_prompt_3_results,\n",
    "                                gpt3_5_tot_detailed_prompt_3_results]))\n",
    "\n",
    "# Total zero-shot\n",
    "print(\"Zero-Shot\" + extract_fails([gpt4o_zs_baseline_system_prompt_results,\n",
    "                                gpt4o_zs_baseline_system_prompt_detailed_results,\n",
    "                                gpt4_zs_baseline_system_prompt_results,\n",
    "                                gpt4_zs_baseline_system_prompt_detailed_results,\n",
    "                                gpt3_5_zs_baseline_system_prompt_results,\n",
    "                                gpt3_5_zs_baseline_system_prompt_detailed_results]))\n",
    "\n",
    "# Total role-based\n",
    "print(\"Role-Based\" + extract_fails([gpt4o_zs_role_system_prompt_results,\n",
    "                                gpt4o_zs_role_system_prompt_detailed_results,\n",
    "                                gpt4o_zs_role_best_system_prompt_results,\n",
    "                                gpt4o_zs_role_best_system_prompt_detailed_results,\n",
    "                                gpt4_zs_role_system_prompt_results,\n",
    "                                gpt4_zs_role_system_prompt_detailed_results,\n",
    "                                gpt4_zs_role_best_system_prompt_results,\n",
    "                                gpt4_zs_role_best_system_prompt_detailed_results,\n",
    "                                gpt3_5_zs_role_system_prompt_results,\n",
    "                                gpt3_5_zs_role_system_prompt_detailed_results,\n",
    "                                gpt3_5_zs_role_best_system_prompt_results,\n",
    "                                gpt3_5_zs_role_best_system_prompt_detailed_results]))\n",
    "\n",
    "# Total cot\n",
    "print(\"Zero-Shot CoT\" + extract_fails([gpt4o_cot_human_prompt_results,\n",
    "                                gpt4o_cot_human_prompt_detailed_results,\n",
    "                                gpt4o_cot_ape_prompt_results,\n",
    "                                gpt4o_cot_ape_prompt_detailed_results,\n",
    "                                gpt4_cot_human_prompt_results,\n",
    "                                gpt4_cot_human_prompt_detailed_results,\n",
    "                                gpt4_cot_ape_prompt_results,\n",
    "                                gpt4_cot_ape_prompt_detailed_results,\n",
    "                                gpt3_5_cot_human_prompt_results,\n",
    "                                gpt3_5_cot_human_prompt_detailed_results,\n",
    "                                gpt3_5_cot_ape_prompt_results,\n",
    "                                gpt3_5_cot_ape_prompt_detailed_results]))\n",
    "\n",
    "# Total meta\n",
    "print(\"Meta prompting\" + extract_fails([gpt4o_meta_system_prompt_results,\n",
    "                                gpt4o_meta_system_prompt_detailed_results,\n",
    "                                gpt4_meta_system_prompt_results,\n",
    "                                gpt4_meta_system_prompt_detailed_results,\n",
    "                                gpt3_5_meta_system_prompt_results,\n",
    "                                gpt3_5_meta_system_prompt_detailed_results]))\n",
    "\n",
    "# Total meta-meta\n",
    "print(\"Meta meta prompting\" + extract_fails([gpt4o_meta_meta_prompt_results,\n",
    "                                gpt4o_meta_meta_prompt_detailed_results,\n",
    "                                gpt4_meta_meta_prompt_results,\n",
    "                                gpt4_meta_meta_prompt_detailed_results,\n",
    "                                gpt3_5_meta_meta_prompt_results,\n",
    "                                gpt3_5_meta_meta_prompt_detailed_results]))\n",
    "\n",
    "# Total tot\n",
    "print(\"ToT-style prompts\" + extract_fails([gpt4o_tot_prompt_1_results,\n",
    "                                gpt4o_tot_detailed_prompt_1_results,\n",
    "                                gpt4o_tot_prompt_2_results,\n",
    "                                gpt4o_tot_detailed_prompt_2_results,\n",
    "                                gpt4o_tot_prompt_3_results,\n",
    "                                gpt4o_tot_detailed_prompt_3_results,\n",
    "                                gpt4_tot_prompt_1_results,\n",
    "                                gpt4_tot_detailed_prompt_1_results,\n",
    "                                gpt4_tot_prompt_2_results,\n",
    "                                gpt4_tot_detailed_prompt_2_results,\n",
    "                                gpt4_tot_prompt_3_results,\n",
    "                                gpt4_tot_detailed_prompt_3_results,\n",
    "                                gpt3_5_tot_prompt_1_results,\n",
    "                                gpt3_5_tot_detailed_prompt_1_results,\n",
    "                                gpt3_5_tot_prompt_2_results,\n",
    "                                gpt3_5_tot_detailed_prompt_2_results,\n",
    "                                gpt3_5_tot_prompt_3_results,\n",
    "                                gpt3_5_tot_detailed_prompt_3_results]))\n",
    "\n",
    "# Total detailed\n",
    "print(\"Total detailed prompts\" + extract_fails([gpt4o_zs_baseline_system_prompt_detailed_results,\n",
    "                                gpt4o_zs_role_system_prompt_detailed_results,\n",
    "                                gpt4o_zs_role_best_system_prompt_detailed_results,\n",
    "                                gpt4o_cot_human_prompt_detailed_results,\n",
    "                                gpt4o_cot_ape_prompt_detailed_results,\n",
    "                                gpt4o_meta_system_prompt_detailed_results,\n",
    "                                gpt4o_meta_meta_prompt_detailed_results,\n",
    "                                gpt4o_tot_detailed_prompt_1_results,\n",
    "                                gpt4o_tot_detailed_prompt_2_results,\n",
    "                                gpt4o_tot_detailed_prompt_3_results,\n",
    "                                gpt4_zs_baseline_system_prompt_detailed_results,\n",
    "                                gpt4_zs_role_system_prompt_detailed_results,\n",
    "                                gpt4_zs_role_best_system_prompt_detailed_results,\n",
    "                                gpt4_cot_human_prompt_detailed_results,\n",
    "                                gpt4_cot_ape_prompt_detailed_results,\n",
    "                                gpt4_meta_system_prompt_detailed_results,\n",
    "                                gpt4_meta_meta_prompt_detailed_results,\n",
    "                                gpt4_tot_detailed_prompt_1_results,\n",
    "                                gpt4_tot_detailed_prompt_2_results,\n",
    "                                gpt4_tot_detailed_prompt_3_results,\n",
    "                                gpt3_5_zs_baseline_system_prompt_detailed_results,\n",
    "                                gpt3_5_zs_role_system_prompt_detailed_results,\n",
    "                                gpt3_5_zs_role_best_system_prompt_detailed_results,\n",
    "                                gpt3_5_cot_human_prompt_detailed_results,\n",
    "                                gpt3_5_cot_ape_prompt_detailed_results,\n",
    "                                gpt3_5_meta_system_prompt_detailed_results,\n",
    "                                gpt3_5_meta_meta_prompt_detailed_results,\n",
    "                                gpt3_5_tot_detailed_prompt_1_results,\n",
    "                                gpt3_5_tot_detailed_prompt_2_results,\n",
    "                                gpt3_5_tot_detailed_prompt_3_results]))\n",
    "\n",
    "# Total not detailed\n",
    "print(\"Total not detailed prompts\" + extract_fails([gpt4o_zs_baseline_system_prompt_results,\n",
    "                                gpt4o_zs_role_system_prompt_results,\n",
    "                                gpt4o_zs_role_best_system_prompt_results,\n",
    "                                gpt4o_cot_human_prompt_results,\n",
    "                                gpt4o_cot_ape_prompt_results,\n",
    "                                gpt4o_meta_system_prompt_results,\n",
    "                                gpt4o_meta_meta_prompt_results,\n",
    "                                gpt4o_tot_prompt_1_results,\n",
    "                                gpt4o_tot_prompt_2_results,\n",
    "                                gpt4o_tot_prompt_3_results,\n",
    "                                gpt4_zs_baseline_system_prompt_results,\n",
    "                                gpt4_zs_role_system_prompt_results,\n",
    "                                gpt4_zs_role_best_system_prompt_results,\n",
    "                                gpt4_cot_human_prompt_results,\n",
    "                                gpt4_cot_ape_prompt_results,\n",
    "                                gpt4_meta_system_prompt_results,\n",
    "                                gpt4_meta_meta_prompt_results,\n",
    "                                gpt4_tot_prompt_1_results,\n",
    "                                gpt4_tot_prompt_2_results,\n",
    "                                gpt4_tot_prompt_3_results,\n",
    "                                gpt3_5_zs_baseline_system_prompt_results,\n",
    "                                gpt3_5_zs_role_system_prompt_results,\n",
    "                                gpt3_5_zs_role_best_system_prompt_results,\n",
    "                                gpt3_5_cot_human_prompt_results,\n",
    "                                gpt3_5_cot_ape_prompt_results,\n",
    "                                gpt3_5_meta_system_prompt_results,\n",
    "                                gpt3_5_meta_meta_prompt_results,\n",
    "                                gpt3_5_tot_prompt_1_results,\n",
    "                                gpt3_5_tot_prompt_2_results,\n",
    "                                gpt3_5_tot_prompt_3_results]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_list = [\n",
    "    gpt4o_zs_baseline_system_prompt_results,\n",
    "    gpt4o_zs_baseline_system_prompt_detailed_results,\n",
    "    gpt4o_zs_role_system_prompt_results,\n",
    "    gpt4o_zs_role_system_prompt_detailed_results,\n",
    "    gpt4o_zs_role_best_system_prompt_results,\n",
    "    gpt4o_zs_role_best_system_prompt_detailed_results\n",
    "]\n",
    "\n",
    "results_gpt4o_zs = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt4o_zs = gpt4o_zs_baseline_system_prompt_scores + gpt4o_zs_baseline_system_prompt_detailed_scores + gpt4o_zs_role_system_prompt_scores + gpt4o_zs_role_system_prompt_detailed_scores + gpt4o_zs_role_best_system_prompt_scores + gpt4o_zs_role_best_system_prompt_detailed_scores\n",
    "\n",
    "agg_list = [\n",
    "    gpt4o_cot_human_prompt_results,\n",
    "    gpt4o_cot_human_prompt_detailed_results,\n",
    "    gpt4o_cot_ape_prompt_results,\n",
    "    gpt4o_cot_ape_prompt_detailed_results\n",
    "]\n",
    "\n",
    "results_gpt4o_cot = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt4o_cot = gpt4o_cot_human_prompt_scores + gpt4o_cot_human_prompt_detailed_scores + gpt4o_cot_ape_prompt_scores + gpt4o_cot_ape_prompt_detailed_scores\n",
    "\n",
    "agg_list = [\n",
    "    gpt4o_tot_prompt_1_results,\n",
    "    gpt4o_tot_prompt_2_results,\n",
    "    gpt4o_tot_prompt_3_results,\n",
    "    gpt4o_tot_detailed_prompt_1_results,\n",
    "    gpt4o_tot_detailed_prompt_2_results,\n",
    "    gpt4o_tot_detailed_prompt_3_results\n",
    "]\n",
    "\n",
    "results_gpt4o_tot = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt4o_tot = gpt4o_tot_prompt_1_scores + gpt4o_tot_prompt_2_scores + gpt4o_tot_prompt_3_scores + gpt4o_tot_detailed_prompt_1_scores + gpt4o_tot_detailed_prompt_2_scores + gpt4o_tot_detailed_prompt_3_scores\n",
    "\n",
    "agg_list = [\n",
    "    gpt4o_meta_system_prompt_results,\n",
    "    gpt4o_meta_system_prompt_detailed_results,\n",
    "    gpt4o_meta_meta_prompt_results,\n",
    "    gpt4o_meta_meta_prompt_detailed_results\n",
    "]\n",
    "\n",
    "results_gpt4o_meta = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt4o_meta = gpt4o_meta_system_prompt_scores + gpt4o_meta_system_prompt_detailed_scores + gpt4o_meta_meta_prompt_scores + gpt4o_meta_meta_prompt_detailed_scores\n",
    "\n",
    "agg_list = [\n",
    "    results_gpt4o_zs,\n",
    "    results_gpt4o_cot,\n",
    "    results_gpt4o_tot,\n",
    "    results_gpt4o_meta\n",
    "]\n",
    "\n",
    "results_gpt4o = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt4o = scores_gpt4o_zs + scores_gpt4o_cot + scores_gpt4o_tot + scores_gpt4o_meta\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_list = [\n",
    "    gpt4_zs_baseline_system_prompt_results,\n",
    "    gpt4_zs_baseline_system_prompt_detailed_results,\n",
    "    gpt4_zs_role_system_prompt_results,\n",
    "    gpt4_zs_role_system_prompt_detailed_results,\n",
    "    gpt4_zs_role_best_system_prompt_results,\n",
    "    gpt4_zs_role_best_system_prompt_detailed_results\n",
    "]\n",
    "\n",
    "results_gpt4_zs = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt4_zs = gpt4_zs_baseline_system_prompt_scores + gpt4_zs_baseline_system_prompt_detailed_scores + gpt4_zs_role_system_prompt_scores + gpt4_zs_role_system_prompt_detailed_scores + gpt4_zs_role_best_system_prompt_scores + gpt4_zs_role_best_system_prompt_detailed_scores\n",
    "\n",
    "agg_list = [\n",
    "    gpt4_cot_human_prompt_results,\n",
    "    gpt4_cot_human_prompt_detailed_results,\n",
    "    gpt4_cot_ape_prompt_results,\n",
    "    gpt4_cot_ape_prompt_detailed_results\n",
    "]\n",
    "\n",
    "results_gpt4_cot = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt4_cot = gpt4_cot_human_prompt_scores + gpt4_cot_human_prompt_detailed_scores + gpt4_cot_ape_prompt_scores + gpt4_cot_ape_prompt_detailed_scores\n",
    "\n",
    "agg_list = [\n",
    "    gpt4_tot_prompt_1_results,\n",
    "    gpt4_tot_prompt_2_results,\n",
    "    gpt4_tot_prompt_3_results,\n",
    "    gpt4_tot_detailed_prompt_1_results,\n",
    "    gpt4_tot_detailed_prompt_2_results,\n",
    "    gpt4_tot_detailed_prompt_3_results\n",
    "]\n",
    "\n",
    "results_gpt4_tot = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt4_tot = gpt4_tot_prompt_1_scores + gpt4_tot_prompt_2_scores + gpt4_tot_prompt_3_scores + gpt4_tot_detailed_prompt_1_scores + gpt4_tot_detailed_prompt_2_scores + gpt4_tot_detailed_prompt_3_scores\n",
    "\n",
    "agg_list = [\n",
    "    gpt4_meta_system_prompt_results,\n",
    "    gpt4_meta_system_prompt_detailed_results,\n",
    "    gpt4_meta_meta_prompt_results,\n",
    "    gpt4_meta_meta_prompt_detailed_results\n",
    "]\n",
    "\n",
    "results_gpt4_meta = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt4_meta = gpt4_meta_system_prompt_scores + gpt4_meta_system_prompt_detailed_scores + gpt4_meta_meta_prompt_scores + gpt4_meta_meta_prompt_detailed_scores\n",
    "\n",
    "agg_list = [\n",
    "    results_gpt4_zs,\n",
    "    results_gpt4_cot,\n",
    "    results_gpt4_tot,\n",
    "    results_gpt4_meta\n",
    "]\n",
    "\n",
    "results_gpt4 = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt4 = scores_gpt4_zs + scores_gpt4_cot + scores_gpt4_tot + scores_gpt4_meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_list = [\n",
    "    gpt3_5_zs_baseline_system_prompt_results,\n",
    "    gpt3_5_zs_baseline_system_prompt_detailed_results,\n",
    "    gpt3_5_zs_role_system_prompt_results,\n",
    "    gpt3_5_zs_role_system_prompt_detailed_results,\n",
    "    gpt3_5_zs_role_best_system_prompt_results,\n",
    "    gpt3_5_zs_role_best_system_prompt_detailed_results\n",
    "]\n",
    "\n",
    "results_gpt3_5_zs = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt3_5_zs = gpt3_5_zs_baseline_system_prompt_scores + gpt3_5_zs_baseline_system_prompt_detailed_scores + gpt3_5_zs_role_system_prompt_scores + gpt3_5_zs_role_system_prompt_detailed_scores + gpt3_5_zs_role_best_system_prompt_scores + gpt3_5_zs_role_best_system_prompt_detailed_scores\n",
    "\n",
    "agg_list = [\n",
    "    gpt3_5_cot_human_prompt_results,\n",
    "    gpt3_5_cot_human_prompt_detailed_results,\n",
    "    gpt3_5_cot_ape_prompt_results,\n",
    "    gpt3_5_cot_ape_prompt_detailed_results\n",
    "]\n",
    "\n",
    "results_gpt3_5_cot = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt3_5_cot = gpt3_5_cot_human_prompt_scores + gpt3_5_cot_human_prompt_detailed_scores + gpt3_5_cot_ape_prompt_scores + gpt3_5_cot_ape_prompt_detailed_scores\n",
    "\n",
    "agg_list = [\n",
    "    gpt3_5_tot_prompt_1_results,\n",
    "    gpt3_5_tot_prompt_2_results,\n",
    "    gpt3_5_tot_prompt_3_results,\n",
    "    gpt3_5_tot_detailed_prompt_1_results,\n",
    "    gpt3_5_tot_detailed_prompt_2_results,\n",
    "    gpt3_5_tot_detailed_prompt_3_results\n",
    "]\n",
    "\n",
    "results_gpt3_5_tot = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt3_5_tot = gpt3_5_tot_prompt_1_scores + gpt3_5_tot_prompt_2_scores + gpt3_5_tot_prompt_3_scores + gpt3_5_tot_detailed_prompt_1_scores + gpt3_5_tot_detailed_prompt_2_scores + gpt3_5_tot_detailed_prompt_3_scores\n",
    "\n",
    "agg_list = [\n",
    "    gpt3_5_meta_system_prompt_results,\n",
    "    gpt3_5_meta_system_prompt_detailed_results,\n",
    "    gpt3_5_meta_meta_prompt_results,\n",
    "    gpt3_5_meta_meta_prompt_detailed_results\n",
    "]\n",
    "\n",
    "results_gpt3_5_meta = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt3_5_meta = gpt3_5_meta_system_prompt_scores + gpt3_5_meta_system_prompt_detailed_scores + gpt3_5_meta_meta_prompt_scores + gpt3_5_meta_meta_prompt_detailed_scores\n",
    "\n",
    "agg_list = [\n",
    "    results_gpt3_5_zs,\n",
    "    results_gpt3_5_cot,\n",
    "    results_gpt3_5_tot,\n",
    "    results_gpt3_5_meta\n",
    "]\n",
    "\n",
    "results_gpt3_5 = aggregate_all_test_results(agg_list)\n",
    "\n",
    "scores_gpt3_5 = scores_gpt3_5_zs + scores_gpt3_5_cot + scores_gpt3_5_tot + scores_gpt3_5_meta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Print aggregated results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GPT-4o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregated Results: GPT-4o\n",
      "{\n",
      "    \"total_responses\": 1000,\n",
      "    \"secrets_used\": 779,\n",
      "    \"base64_encoding_needed\": 145,\n",
      "    \"yaml_not_wrapped\": 2,\n",
      "    \"invalid_yaml\": 9,\n",
      "    \"kubeconform_failed\": 42,\n",
      "    \"no_mysql_or_wordpress\": 12,\n",
      "    \"deployment_failed\": 7,\n",
      "    \"duplicate_resources\": 4,\n",
      "    \"healthy\": 611,\n",
      "    \"unhealthy\": 317,\n",
      "    \"PVC_failed\": 12,\n",
      "    \"Missing_resources\": 3,\n",
      "    \"Missing_key\": 0,\n",
      "    \"MySQL_PVC_unbound\": 4,\n",
      "    \"MySQL_not_ready\": 3,\n",
      "    \"Wordpress_PVC_unbound\": 0,\n",
      "    \"Wordpress_not_ready\": 280,\n",
      "    \"Service_failed\": 3,\n",
      "    \"Unknown_error\": 12\n",
      "}\n",
      "Polaris Scores: [40, 40, 37, 37, 35, 36, 40, 40, 40, 40, 37, 35, 37, 40, 35, 35, 40, 37, 35, 35, 43, 35, 40, 40, 40, 42, 42, 42, 44, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 44, 42, 44, 44, 42, 42, 39, 44, 42, 42, 42, 42, 42, 39, 44, 42, 44, 42, 42, 42, 42, 42, 44, 44, 39, 42, 42, 42, 42, 40, 35, 35, 40, 40, 40, 40, 41, 40, 35, 35, 35, 40, 38, 40, 40, 35, 40, 37, 40, 40, 36, 40, 38, 42, 44, 40, 42, 42, 39, 42, 42, 44, 44, 42, 42, 42, 42, 42, 44, 42, 42, 42, 42, 42, 42, 42, 42, 42, 44, 42, 44, 42, 42, 42, 44, 44, 42, 42, 42, 44, 42, 44, 42, 42, 42, 42, 36, 40, 37, 35, 40, 40, 40, 40, 35, 37, 35, 40, 40, 40, 41, 40, 40, 35, 41, 40, 40, 40, 37, 43, 40, 35, 37, 40, 40, 40, 44, 44, 42, 42, 42, 42, 42, 39, 42, 42, 42, 42, 42, 42, 42, 44, 42, 44, 44, 44, 42, 42, 44, 42, 42, 44, 42, 42, 44, 44, 44, 42, 42, 44, 44, 44, 42, 42, 44, 42, 40, 35, 40, 35, 35, 40, 39, 40, 43, 35, 40, 40, 35, 40, 40, 40, 40, 43, 40, 35, 40, 40, 40, 40, 35, 40, 43, 40, 40, 35, 40, 40, 40, 40, 42, 42, 43, 42, 42, 42, 44, 42, 42, 42, 42, 42, 42, 40, 42, 42, 42, 42, 42, 42, 40, 42, 44, 44, 42, 42, 42, 44, 42, 44, 42, 42, 42, 42, 42, 40, 40, 40, 35, 40, 40, 40, 43, 40, 40, 40, 35, 40, 40, 35, 40, 40, 40, 40, 40, 40, 37, 40, 40, 40, 35, 40, 35, 43, 40, 40, 40, 42, 42, 42, 42, 44, 42, 44, 40, 39, 42, 42, 42, 42, 42, 40, 42, 42, 42, 42, 42, 42, 44, 42, 42, 42, 44, 42, 44, 42, 40, 44, 44, 42, 42, 40, 43, 40, 35, 43, 35, 37, 35, 35, 37, 37, 35, 40, 40, 35, 35, 43, 37, 37, 40, 35, 40, 40, 40, 40, 35, 40, 40, 37, 40, 40, 40, 40, 35, 40, 35, 35, 40, 37, 35, 35, 35, 43, 40, 35, 35, 37, 43, 40, 35, 40, 40, 35, 37, 37, 37, 43, 40, 40, 40, 40, 40, 37, 35, 40, 35, 37, 37, 37, 40, 40, 37, 37, 35, 40, 40, 37, 42, 42, 44, 44, 44, 44, 42, 42, 44, 42, 42, 42, 42, 42, 42, 44, 44, 42, 44, 42, 42, 42, 42, 42, 42, 43, 42, 44, 42, 42, 44, 44, 42, 44, 39, 42, 42, 42, 42, 42, 44, 42, 42, 42, 42, 42, 42, 42, 40, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 44, 42, 44, 42, 42, 42, 42, 39, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 39, 42, 42, 42, 42, 42, 42, 44, 42, 42, 42, 42, 42, 42, 40, 37, 40, 40, 40, 40, 43, 40, 40, 43, 35, 40, 40, 40, 43, 35, 37, 35, 43, 40, 40, 43, 43, 40, 35, 40, 40, 40, 44, 44, 42, 42, 42, 42, 42, 42, 42, 42, 44, 42, 42, 42, 44, 44, 42, 42, 42, 42, 42, 42, 44, 44, 42, 42, 44, 39, 42, 39, 44, 42, 42, 42, 42, 42, 42, 44, 42, 42, 42, 42, 42, 44, 55, 62, 58, 57, 59, 60, 44, 51, 46, 59, 45, 46, 54, 46, 51, 61, 56, 41]\n"
     ]
    }
   ],
   "source": [
    "print(f\"Aggregated Results: GPT-4o\")\n",
    "print(json.dumps(results_gpt4o, indent=4))\n",
    "print(f\"Polaris Scores: {scores_gpt4o}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}