00_PDF解析与QA抽取_v1.ipynb

{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "这份代码在QA打分的环节，prompt做了更新，大家可以参考新版本，新版本文件名为`00_PDF解析与QA抽取_v1.1.ipynb`",
   "id": "309aca4f7fdedf25"
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1f414496-9550-494b-9938-0add9cdedbb1",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1f414496-9550-494b-9938-0add9cdedbb1",
    "outputId": "5e6f9b0d-e207-4166-dd86-e674741d7144",
    "scrolled": true
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Requirement already satisfied: langchain in /usr/local/lib/python3.10/dist-packages (0.2.8)\n",
      "Requirement already satisfied: langchain_community in /usr/local/lib/python3.10/dist-packages (0.2.7)\n",
      "Requirement already satisfied: pypdf in /usr/local/lib/python3.10/dist-packages (4.3.0)\n",
      "Collecting openai\n",
      "  Downloading openai-1.35.14-py3-none-any.whl (328 kB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m328.5/328.5 kB\u001B[0m \u001B[31m2.4 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.31)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.9.5)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (4.0.3)\n",
      "Requirement already satisfied: langchain-core<0.3.0,>=0.2.19 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.2.19)\n",
      "Requirement already satisfied: langchain-text-splitters<0.3.0,>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.2.2)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.17 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.1.86)\n",
      "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.25.2)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.8.2)\n",
      "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.31.0)\n",
      "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.5.0)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /usr/local/lib/python3.10/dist-packages (from langchain_community) (0.6.7)\n",
      "Requirement already satisfied: typing_extensions>=4.0 in /usr/local/lib/python3.10/dist-packages (from pypdf) (4.12.2)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai) (3.7.1)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai) (1.7.0)\n",
      "Collecting httpx<1,>=0.23.0 (from openai)\n",
      "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m75.6/75.6 kB\u001B[0m \u001B[31m9.8 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai) (1.3.1)\n",
      "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.4)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.4)\n",
      "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (3.7)\n",
      "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (1.2.1)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain_community) (3.21.3)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain_community) (0.9.0)\n",
      "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (2024.7.4)\n",
      "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai)\n",
      "  Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m77.9/77.9 kB\u001B[0m \u001B[31m8.5 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai)\n",
      "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m58.3/58.3 kB\u001B[0m \u001B[31m6.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: jsonpatch<2.0,>=1.33 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.3.0,>=0.2.19->langchain) (1.33)\n",
      "Requirement already satisfied: packaging<25,>=23.2 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.3.0,>=0.2.19->langchain) (24.1)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /usr/local/lib/python3.10/dist-packages (from langsmith<0.2.0,>=0.1.17->langchain) (3.10.6)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (2.20.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.3.2)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2.0.7)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (3.0.3)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.10/dist-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.3.0,>=0.2.19->langchain) (3.0.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain_community) (1.0.0)\n",
      "Installing collected packages: h11, httpcore, httpx, openai\n",
      "Successfully installed h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 openai-1.35.14\n"
     ]
    }
   ],
   "source": [
    "!pip install langchain langchain_community pypdf openai"
   ]
  },
  {
   "cell_type": "code",
   "id": "37d1b8b1-d68e-4151-ac78-2265871095b4",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "37d1b8b1-d68e-4151-ac78-2265871095b4",
    "outputId": "27ecf6ec-8a83-4771-80d9-ecef3049cabc",
    "jupyter": {
     "is_executing": true
    }
   },
   "source": [
    "%env API_KEY=\n",
    "%env BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "dZ7wJ4jH62js",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dZ7wJ4jH62js",
    "outputId": "e9b9307f-9fb6-4549-da81-9e921d61f5a8"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "langchain           0.2.8\n",
      "langchain_community 0.2.7\n",
      "pypdf               4.3.0\n",
      "openai              1.35.14\n"
     ]
    }
   ],
   "source": [
    "import langchain, langchain_community, pypdf, openai\n",
    "\n",
    "for module in (langchain, langchain_community, pypdf, openai):\n",
    "    print(f\"{module.__name__:<20}{module.__version__}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6a1316f-07b1-490c-a673-05c73c3a6124",
   "metadata": {
    "id": "b6a1316f-07b1-490c-a673-05c73c3a6124"
   },
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import PyPDFLoader\n",
    "from langchain.schema import Document\n",
    "\n",
    "from datetime import datetime\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "018cdaf6-f8af-4879-9153-b0686782fb34",
   "metadata": {
    "id": "018cdaf6-f8af-4879-9153-b0686782fb34"
   },
   "outputs": [],
   "source": [
    "dt = '20240713'\n",
    "version = 'v1'\n",
    "\n",
    "output_dir = os.path.join('outputs', f'{version}_{dt}')\n",
    "os.makedirs(output_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2c81807d-e4f5-4618-8e7b-688dd72b8f0c",
   "metadata": {
    "id": "2c81807d-e4f5-4618-8e7b-688dd72b8f0c"
   },
   "outputs": [],
   "source": [
    "loader = PyPDFLoader(\"data/2024全球经济金融展望报告.pdf\")\n",
    "documents = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0e5f0196-a4c5-44ab-9af1-a74a6a62d5fd",
   "metadata": {
    "id": "0e5f0196-a4c5-44ab-9af1-a74a6a62d5fd",
    "outputId": "a2ca700c-86cb-4918-fc3b-448c9585a9c0",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "53"
      ]
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "len(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4a8a771a-e9ea-46df-a911-8a1f32c9198c",
   "metadata": {
    "id": "4a8a771a-e9ea-46df-a911-8a1f32c9198c",
    "outputId": "0eab3ae2-6e43-408d-bac0-1b8c6e1de9d5",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "643"
      ]
     },
     "metadata": {},
     "execution_count": 14
    }
   ],
   "source": [
    "len(documents[0].page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c2bc3e04-9737-4e6b-b292-3b582b5e0c99",
   "metadata": {
    "id": "c2bc3e04-9737-4e6b-b292-3b582b5e0c99"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "pattern = r\"^全球经济金融展望报告\\n中国银行研究院 \\d+ 2024年\"\n",
    "# re.sub(pattern, '', splitted_docs_large[3].page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4eae10fa-0430-474b-b6ba-deac98838fba",
   "metadata": {
    "id": "4eae10fa-0430-474b-b6ba-deac98838fba"
   },
   "outputs": [],
   "source": [
    "merged_docs = [Document(page_content='\\n'.join(re.sub(pattern, '', doc.page_content) for doc in documents))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2a08d8be-029b-4dd8-a741-985e38529a66",
   "metadata": {
    "id": "2a08d8be-029b-4dd8-a741-985e38529a66"
   },
   "outputs": [],
   "source": [
    "from uuid import uuid4\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "def split_docs(documents, filepath, chunk_size=400, chunk_overlap=40, seperators=['\\n\\n\\n', '\\n\\n'], force_split=False):\n",
    "    if os.path.exists(filepath) and not force_split:\n",
    "        print('found cache, restoring...')\n",
    "        return pickle.load(open(filepath, 'rb'))\n",
    "\n",
    "    splitter = RecursiveCharacterTextSplitter(\n",
    "        chunk_size=chunk_size,\n",
    "        chunk_overlap=chunk_overlap,\n",
    "        separators=seperators\n",
    "    )\n",
    "    split_docs = splitter.split_documents(documents)\n",
    "    for chunk in split_docs:\n",
    "        chunk.metadata['uuid'] = str(uuid4())\n",
    "\n",
    "    pickle.dump(split_docs, open(filepath, 'wb'))\n",
    "\n",
    "    return split_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9113d9a5-7d7c-4e3a-876c-f8a69cd7d8a2",
   "metadata": {
    "id": "9113d9a5-7d7c-4e3a-876c-f8a69cd7d8a2"
   },
   "outputs": [],
   "source": [
    "splitted_docs = split_docs(documents, os.path.join(output_dir, 'split_docs.pkl'), chunk_size=500, chunk_overlap=50)\n",
    "splitted_docs_large = split_docs(merged_docs, os.path.join(output_dir, 'split_docs_large.pkl'), chunk_size=1500, chunk_overlap=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b4989f2c-4474-4612-a470-ade89cb3a9b4",
   "metadata": {
    "id": "b4989f2c-4474-4612-a470-ade89cb3a9b4",
    "outputId": "00b1207c-ba79-435e-dfa0-9e6e09aca496",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(52, 28)"
      ]
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "len(splitted_docs), len(splitted_docs_large)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "13fe597d-120e-47a1-8ec1-a167fea1d328",
   "metadata": {
    "id": "13fe597d-120e-47a1-8ec1-a167fea1d328"
   },
   "outputs": [],
   "source": [
    "uuid2doc = {doc.metadata['uuid']: doc.page_content for doc in splitted_docs}\n",
    "uuid2large_doc = {doc.metadata['uuid']: doc.page_content for doc in splitted_docs_large}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b9be7ac-547e-4347-acc8-193b09e6277e",
   "metadata": {
    "id": "0b9be7ac-547e-4347-acc8-193b09e6277e"
   },
   "source": [
    "# 抽取QA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "976d80d0-d5b2-486f-83ca-f2a194b07b6b",
   "metadata": {
    "id": "976d80d0-d5b2-486f-83ca-f2a194b07b6b"
   },
   "outputs": [],
   "source": [
    "qa_gen_prompt_tmpl = \"\"\"\n",
    "我会给你一段文本（<document></document>之间的部分），你需要阅读这段文本，分别针对这段文本生成8个问题、用户回答这个问题的上下文，和基于上下文对问题的回答。\n",
    "\n",
    "对问题、上下文、答案的要求：\n",
    "\n",
    "问题要与这段文本相关，不要询问类似“这个问题的答案在哪一章”这样的问题\n",
    "上下文：上下文必须与原始文本的内容保持一致，不要进行缩写、扩写、改写、摘要、替换词语等\n",
    "答案：回答请保持完整且简洁，无须重复问题。答案要能够独立回答问题，而不是引用现有的章节、页码等\n",
    "\n",
    "返回结果以JSON形式组织，格式为[{\"question\": \"...\", \"context\": ..., \"answer\": \"...\"}, ...]。\n",
    "如果当前文本主要是目录，或者是一些人名、地址、电子邮箱等没有办法生成有意义的问题时，可以返回[]。\n",
    "\n",
    "下方是文本：\n",
    "<document>\n",
    "{{document}}\n",
    "</document>\n",
    "\n",
    "请生成结果：\n",
    "\"\"\"\n",
    "\n",
    "qa_gen_prompt_tmpl_large_context = \"\"\"\n",
    "我会给你一段文本（<document></document>之间的部分），你需要阅读这段文本，分别针对这段文本生成2个问题，和基于这段文本对问题的回答，回答请保持完整，无须重复问题。\n",
    "尽可能创建一些需要综合*大段*文本才能回答的问题，但不要问类似“这一段主要讲了什么内容”这样的问题，答案要能够独立回答问题，而不是引用现有的章节、页码等；不要问具体过于细节的问题，例如“海湾国家的2024年预期经济增长率是多少”，而是尽可能问类似“2024年全球经济的几大趋势是什么”、“受局部中东地区紧张局势影响，可能对全球原物料有哪些影响”。\n",
    "返回结果以JSON形式组织，格式为[{\"question\": \"...\", \"answer\": \"...\"}, ...]。\n",
    "如果当前文本主要是目录，或者是一些人名、地址、电子邮箱等没有办法生成有意义的问题时，可以返回[]。\n",
    "\n",
    "下方是文本：\n",
    "<document>\n",
    "{{document}}\n",
    "</document>\n",
    "\n",
    "请生成结果：\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2938667e-aa5e-43d0-80ba-8b80b66c43ee",
   "metadata": {
    "id": "2938667e-aa5e-43d0-80ba-8b80b66c43ee"
   },
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "import time\n",
    "import random\n",
    "\n",
    "client = OpenAI(\n",
    "    api_key=os.environ['API_KEY'],\n",
    "    base_url=os.environ['BASE_URL']\n",
    ")\n",
    "\n",
    "def build_qa_prompt(prompt_tmpl, text):\n",
    "    prompt = prompt_tmpl.replace('{{document}}', text).strip()\n",
    "    return prompt\n",
    "\n",
    "def chat(prompt, max_retry=3, debug=False, top_p=0.95, temperature=0.85):\n",
    "    def do_chat(prompt):\n",
    "        completion = client.chat.completions.create(\n",
    "            model='qwen-long',\n",
    "            messages=[\n",
    "                # {\"role\": \"system\", \"content\": \"你是一个有用的人工智能助手\"},\n",
    "                {\"role\": \"user\", \"content\": prompt}\n",
    "            ],\n",
    "            top_p=top_p,\n",
    "            temperature=temperature\n",
    "        )\n",
    "        return completion.choices[0].message.content\n",
    "\n",
    "    while max_retry > 0:\n",
    "        try:\n",
    "            return do_chat(prompt)\n",
    "        except Exception as e:\n",
    "            max_retry -= 1\n",
    "            sleep_seconds = random.randint(1, 4)\n",
    "            if debug:\n",
    "                print(f\"{str(e)}, remain retry: {max_retry}, sleeping {sleep_seconds}s {prompt}\")\n",
    "            time.sleep(sleep_seconds)\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "aabdd8b4-1cea-4d2f-89b3-0f859a65dd72",
   "metadata": {
    "id": "aabdd8b4-1cea-4d2f-89b3-0f859a65dd72"
   },
   "outputs": [],
   "source": [
    "text = splitted_docs[40].page_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a7bf3d69-4de4-4666-9297-fe5417b591f1",
   "metadata": {
    "id": "a7bf3d69-4de4-4666-9297-fe5417b591f1",
    "outputId": "4647a5ef-7a74-4d6a-ff30-40e4c6b2196b",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "全球经济金融展望报告\n",
      "中国银行研究院 40 2024年\n",
      "注重绿色经济发展，为中海经贸合作创造了良好机遇。\n",
      "（一）近年来海湾六国经济表现受能源价格影响较大\n",
      "受国际能源价格走势影响，近年来海湾六国经济增长波动较大。2022年，\n",
      "在全球油气价格飙升推动下，海湾六国经济增长强劲，实际GDP增速达7.3%。\n",
      "其中，沙特占据海湾六国经济体量的半壁江山。2022年，沙特GDP规模达1.1\n",
      "万亿美元，阿联酋和卡塔尔GDP规模分别为5075亿和2373亿美元，位列第二\n",
      "和第三位（图26）。油气部门在海湾六国经济总量中的占比超过40%，在沙特\n",
      "的占比更是接近七成。\n",
      "图26：2022年海湾六国经济体量（亿美元）\n",
      "资料来源：世界银行，中国银行研究院\n",
      "2023年以来，海湾六国经济增长明显放缓。主要经济体货币政策加速收紧、\n",
      "全球经济活动减少使全球能源需求下降，海湾六国经济增长面临严重挑战。IMF\n",
      "在最新展望报告中将海湾六国2023年经济增速下调0.8个百分点至1.7%。其中，\n",
      "沙特经济增速预计仅为0.8%，低于中东石油出口国的平均水平。相比之下，阿\n",
      "联酋韧性较强，预计2023年和2024年经济增速分别为3.4%和4.0%，为海湾\n",
      "六国中最高（图27）。鉴于海湾六国经济结构相对单一，非石油部门虽然为海\n"
     ]
    }
   ],
   "source": [
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "494dd058-017d-41fd-90e4-b531db619d8f",
   "metadata": {
    "id": "494dd058-017d-41fd-90e4-b531db619d8f",
    "outputId": "63449c8b-77a3-4ab8-babc-2971a5cf13d2",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "```json\n",
      "[\n",
      "    {\n",
      "        \"question\": \"海湾六国经济增长受什么影响较大？\",\n",
      "        \"context\": \"受国际能源价格走势影响，近年来海湾六国经济增长波动较大。\",\n",
      "        \"answer\": \"国际能源价格走势\"\n",
      "    },\n",
      "    {\n",
      "        \"question\": \"2022年海湾六国的平均GDP增速是多少？\",\n",
      "        \"context\": \"2022年，海湾六国经济增长强劲，实际GDP增速达7.3%。\",\n",
      "        \"answer\": \"7.3%\"\n",
      "    },\n",
      "    {\n",
      "        \"question\": \"哪个国家在海湾六国中GDP规模最大？\",\n",
      "        \"context\": \"沙特占据海湾六国经济体量的半壁江山。2022年，沙特GDP规模达1.1万亿美元\",\n",
      "        \"answer\": \"沙特\"\n",
      "    },\n",
      "    {\n",
      "        \"question\": \"2022年阿联酋的GDP规模大约是多少亿美元？\",\n",
      "        \"context\": \"阿联酋和卡塔尔GDP规模分别为5075亿和2373亿美元，位列第二和第三位\",\n",
      "        \"answer\": \"5075亿\"\n",
      "    },\n",
      "    {\n",
      "        \"question\": \"2022年沙特油气部门在其经济总量中的占比大约是多少？\",\n",
      "        \"context\": \"油气部门在海湾六国经济总量中的占比超过40%，在沙特的占比更是接近七成。\",\n",
      "        \"answer\": \"接近七成\"\n",
      "    },\n",
      "    {\n",
      "        \"question\": \"IMF预测2023年海湾六国的平均经济增速是多少？\",\n",
      "        \"context\": \"IMF在最新展望报告中将海湾六国2023年经济增速下调0.8个百分点至1.7%。\",\n",
      "        \"answer\": \"1.7%\"\n",
      "    },\n",
      "    {\n",
      "        \"question\": \"2023年和2024年阿联酋预计的经济增速是多少？\",\n",
      "        \"context\": \"预计2023年和2024年经济增速分别为3.4%和4.0%，为海湾六国中最高。\",\n",
      "        \"answer\": \"2023年3.4%，2024年4.0%\"\n",
      "    },\n",
      "    {\n",
      "        \"question\": \"海湾六国经济结构的主要问题是什么？\",\n",
      "        \"context\": \"鉴于海湾六国经济结构相对单一，非石油部门虽然为海\",\n",
      "        \"answer\": \"经济结构相对单一\"\n",
      "    }\n",
      "]\n",
      "```\n"
     ]
    }
   ],
   "source": [
    "print(chat(build_qa_prompt(qa_gen_prompt_tmpl, text), debug=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a8465bb6-5f69-4cd1-b957-2ee01ec8cd4e",
   "metadata": {
    "id": "a8465bb6-5f69-4cd1-b957-2ee01ec8cd4e",
    "outputId": "773388e3-8239-416c-cd34-d6cde29c23bb",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "存在专向性1或属于禁止性补贴，且对欧盟内部产业构成实质性损害的情况下，欧盟即可认定补贴属于\n",
      "可诉性补贴。在此情况下，欧盟一般可能采取两类反补贴措施：一是加征反补贴税，针对不同类型的\n",
      "企业，适用的加征税率也有所差异，其中被认定为不合作企业将会被征收最高税率；二是出口商接受\n",
      "价格承诺，以不低于特定价格出口。反补贴措施一般实施五年。当欧盟认定补贴未对其产业带来损害、\n",
      "补贴微量或申请人撤诉的情况下，可以无措施结案。\n",
      "欧盟从2010年首次发起对华反补贴调查开始，共计对华发起过18起反补贴调查，涉及多个制造\n",
      "业领域，其中接近三分之二的调查最终采取了加征关税的措施。因此，从历史经验看，反补贴调查一\n",
      "经发起，欧盟较大概率最终将采取加征关税措施。\n",
      "3.日本经济总体保持稳定，货币政策有望迎来拐点\n",
      "2023年日本经济整体呈现先升后降走势。上半年，在疫后重新开放市场、\n",
      "供应链修复等因素影响下，日本经济实现较快增长，一、二季度GDP环比增长\n",
      "折年率分别达到3.7%和4.5%。但三季度日本经济增长形势急转直下，GDP环\n",
      "比折年率降至-2.1%，三个季度以来首次出现负增长。从全年情况看，通胀压力\n",
      "和疲软的日元抑制了私人需求，占日本GDP近六成的个人消费连续两个季度出\n",
      "现负增长，严重拖累日本经济。前期对日本经济增长贡献较大的外需在三季度\n",
      "也逐渐失去活力，特别是前期增长旺盛的访日外国游客消费在三季度出现5%\n",
      "的环比下滑，同期日本进口出现三个季度的首次正增长，净出口对整体经济的\n",
      "贡献由正转负。民间设备投资在二季度下降1%后三季度再下降0.6%，企业在\n",
      "价格压力不减的情况下继续削减投资。政府消费经历先降后升走势，日本政府\n",
      "财政政策在三季度开始发力，但仍不足以支撑日本经济整体表现。按照平均汇\n",
      "率折算，前三季度日本经济总量被德国赶超，降至全球第四位。\n",
      "1如果补贴授予当局补贴发放明确限定于特定企业，这种补贴即具有专向性。\n",
      "\n",
      "图11：日本GDP环比增长折年率变化情况（%）\n",
      "资料来源：Wind，中国银行研究院\n",
      "日本通胀压力较大。2022年以来，日本通胀持续上涨，并于2023年初达\n",
      "到41年来高位。此后受益于能源进口价格下滑以及政府对抑制电费和燃气费的\n",
      "补贴政策，通胀水平有所回落，但最新数据仍在3%左右的水平震荡，显著高于\n",
      "过去二十多年0.15%的平均水平。日本对海外能源、食品的依赖程度较为突出，\n",
      "特别是能源进口依赖度高达89%，输入性通胀对日本影响较大。上半年进口价\n",
      "格回落助力日本通胀形势有所缓解，进入三季度日本面临的进口价格压力再度\n",
      "上升。在服务分项和劳动力工资上涨支撑下，剔除食品和能源价格的核心通胀\n",
      "保持较强粘性。虽然日本劳动力工资水平整体有所上调，但工资“跑不赢”物\n",
      "价成为常态，个人消费增长受阻，国内市场活力受到抑制，导致内需对于日本\n",
      "经济整体贡献持续为负。\n"
     ]
    }
   ],
   "source": [
    "print(splitted_docs_large[8].page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "228f79ac-c37a-441c-8c06-af02f7c8b119",
   "metadata": {
    "id": "228f79ac-c37a-441c-8c06-af02f7c8b119",
    "outputId": "31736ea8-aa82-4ea9-e686-4c1d273bfbe2",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "```json\n",
      "[\n",
      "  {\n",
      "    \"question\": \"欧盟在认定补贴属于可诉性补贴后，可能会采取哪些措施？\",\n",
      "    \"answer\": \"欧盟在认定补贴属于可诉性补贴后，一般可能采取加征反补贴税和要求出口商接受价格承诺这两种措施。加征的反补贴税税率根据企业类型不同而有所差异，不合作企业会面临最高税率。价格承诺则意味着出口商必须同意以不低于特定价格出口商品。这些措施通常会实施五年。\"\n",
      "  },\n",
      "  {\n",
      "    \"question\": \"2023年日本经济的走势如何？\",\n",
      "    \"answer\": \"2023年日本经济呈现先升后降的走势。上半年，由于疫后市场重新开放、供应链修复等因素，GDP环比增长强劲，但到了三季度，GDP环比折年率转为负增长，出现负2.1%的增长。通胀压力、疲软的日元抑制了私人需求，个人消费连续两个季度下滑，外需也失去了活力，尤其是访日外国游客消费减少，净出口对经济的贡献由正转负。尽管政府财政政策在三季度开始发力，但经济总量仍被德国赶超，降至全球第四位。\"\n",
      "  }\n",
      "]\n",
      "```\n"
     ]
    }
   ],
   "source": [
    "print(chat(build_qa_prompt(qa_gen_prompt_tmpl_large_context, splitted_docs_large[8].page_content)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "8d6e499b-effc-4d6a-b79e-05db1727fa2f",
   "metadata": {
    "id": "8d6e499b-effc-4d6a-b79e-05db1727fa2f"
   },
   "outputs": [],
   "source": [
    "import threading\n",
    "import concurrent.futures\n",
    "from tqdm.auto import tqdm\n",
    "import json\n",
    "\n",
    "def gen_qa(splitted_docs, prompt_tmpl, qa_ckpt_filename):\n",
    "    qa_ckpt = {}\n",
    "    if os.path.exists(qa_ckpt_filename):\n",
    "        qa_ckpt = open(qa_ckpt_filename).readlines()\n",
    "        qa_ckpt = [json.loads(line.strip()) for line in qa_ckpt if line.strip() != '']\n",
    "        qa_ckpt = {item['uuid']: item for item in qa_ckpt}\n",
    "        print(f'found checkpoint, item count: {len(qa_ckpt)}')\n",
    "\n",
    "    file_lock = threading.Lock()\n",
    "    max_workers = 4\n",
    "    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:\n",
    "        futures = {doc.metadata['uuid']: executor.submit(chat, build_qa_prompt(prompt_tmpl, doc.page_content), 3, True) for doc in splitted_docs if len(doc.page_content.replace('\\n', '')) >= 150 and doc.metadata['uuid'] not in qa_ckpt}\n",
    "        for uuid in tqdm(futures):\n",
    "            future = futures[uuid]\n",
    "            result = future.result()\n",
    "            if result is None:\n",
    "                continue\n",
    "\n",
    "            item = {'uuid': uuid, 'raw_resp': result}\n",
    "            qa_ckpt[uuid] = item\n",
    "\n",
    "            # global file_lock\n",
    "            file_lock.acquire()\n",
    "\n",
    "            try:\n",
    "                with open(qa_ckpt_filename, 'a') as f:\n",
    "                    f.write(json.dumps(item, ensure_ascii=False) + '\\n')\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "            finally:\n",
    "                file_lock.release()\n",
    "    return qa_ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "9525aa93-7474-49a8-a995-00c94de7f313",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 49,
     "referenced_widgets": [
      "97721fe8e229406daa546d30f6b1f3c4",
      "3a1483b980ff40a4b113c9adb1debde7",
      "14f5510c4b9a41f7974b8ef701fe2ac2",
      "070d1468c7264d2d98172e1d75acd2f4",
      "f64c0f996f474cd4ab55b9faa4973f2d",
      "07bd5384c9314c46aa63d2950d655f9a",
      "7726409007ab4afdb345b00002dc0b00",
      "b9fb90215c684f2d8cdbc80ac81ba9e2",
      "a4ccc283659d4794a6ef436a9ec372e1",
      "a5b98dd061c9476b897e7f6df03d285d",
      "2c24223263dd4df1b5305dc878358fff"
     ]
    },
    "id": "9525aa93-7474-49a8-a995-00c94de7f313",
    "outputId": "68997084-1c28-40b6-e62c-39e521ebb4b3"
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "  0%|          | 0/49 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "97721fe8e229406daa546d30f6b1f3c4"
      }
     },
     "metadata": {}
    }
   ],
   "source": [
    "detailed_qa_dict = gen_qa(splitted_docs, qa_gen_prompt_tmpl, os.path.join(output_dir, f\"qa_ckpt_detailed.jsonl\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "72370d3e-5c8b-4012-8c34-fb934196a7d7",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 49,
     "referenced_widgets": [
      "513768f81b094f358168994ad03f9cc2",
      "219d202b5f31479988b4dd52010bdd6e",
      "72376297dce54eb2a47e4526e6906f88",
      "2cd71a6e921f4073bc415fb31f7b63fc",
      "b0c2870b0f014c4793a232bbbb2f1e03",
      "e02a2b231add4980a5d6e52244902ff0",
      "5c222bdd56054b69b708eda14f4cce51",
      "e67f4a688ebe490e9d0e866252136885",
      "d887ab0146b94f399d413b68bc404e3d",
      "c369641245b14114a3c621f6969ccfcd",
      "1d559d3c3b46467b8b6d6cc8193d6b5b"
     ]
    },
    "id": "72370d3e-5c8b-4012-8c34-fb934196a7d7",
    "outputId": "15cb7832-a398-49e8-fc68-3be503e65da2"
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "  0%|          | 0/28 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "513768f81b094f358168994ad03f9cc2"
      }
     },
     "metadata": {}
    }
   ],
   "source": [
    "large_context_qa_dict = gen_qa(splitted_docs_large, qa_gen_prompt_tmpl_large_context, os.path.join(output_dir, f\"qa_ckpt_large_context.jsonl\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fa878e7-9c9f-4700-864f-b9547a5f344d",
   "metadata": {
    "id": "7fa878e7-9c9f-4700-864f-b9547a5f344d"
   },
   "source": [
    "# 后置处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c3a8af8-7af6-44ca-bed8-f386f9f194db",
   "metadata": {
    "id": "0c3a8af8-7af6-44ca-bed8-f386f9f194db"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def convert2json(text):\n",
    "    pattern = r'\\[.*\\]'\n",
    "\n",
    "    text = text.replace('>>>', '')\n",
    "    try:\n",
    "        return json.loads(text)\n",
    "    except:\n",
    "        match = re.search(pattern, text, re.DOTALL)\n",
    "        try:\n",
    "            matched = match.group(0)\n",
    "            return json.loads(matched)\n",
    "        except Exception as e:\n",
    "            print(f\"{match}, {str(e)}\")\n",
    "\n",
    "    return []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "564b7782-57f8-4b3c-9daf-674f0a06bd2a",
   "metadata": {
    "id": "564b7782-57f8-4b3c-9daf-674f0a06bd2a"
   },
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "import pandas as pd\n",
    "import pickle\n",
    "\n",
    "def build_qa_df(qa_ckpt, uuid2doc_map):\n",
    "    data = []\n",
    "\n",
    "    for key, value in tqdm(qa_ckpt.items()):\n",
    "        text = value['raw_resp']\n",
    "        qa_list = convert2json(text)\n",
    "\n",
    "        for item in qa_list:\n",
    "            question = item.get('question', '').strip()\n",
    "            answer = item.get('answer', '').strip()\n",
    "            context = item.get('context', '').strip()\n",
    "\n",
    "            if question == '' or answer == '':\n",
    "                print(qa_list)\n",
    "                continue\n",
    "            data.append({\n",
    "                'uuid': key,\n",
    "                'question': question,\n",
    "                'answer': answer,\n",
    "                'context': context,\n",
    "                'doc': uuid2doc_map[key]\n",
    "            })\n",
    "    qa_df = pd.DataFrame(data)\n",
    "    return qa_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39d9677f-faa0-4a31-9120-2b3ba6427e60",
   "metadata": {
    "colab": {
     "referenced_widgets": [
      "f17d9286affc46a18c185e47a2f5c096",
      "0a8af6d4ba31495b8bab5bccb973976d"
     ]
    },
    "id": "39d9677f-faa0-4a31-9120-2b3ba6427e60",
    "outputId": "30c88821-d33c-462d-81e0-aa4c372dff6a"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f17d9286affc46a18c185e47a2f5c096",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/49 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(8, 1308), match='[\\n    {\\n        \"question\": \"什么是可诉性补贴？\",\\n     >, Invalid control character at: line 5 column 63 (char 177)\n",
      "<re.Match object; span=(8, 1630), match='[\\n    {\\n        \"question\": \"2024年主要经济体的货币市场利率将>, Expecting value: line 10 column 19 (char 433)\n",
      "<re.Match object; span=(8, 1288), match='[\\n    {\\n        \"question\": \"2023年全球大宗商品价格的走势如何>, Invalid control character at: line 5 column 43 (char 144)\n",
      "<re.Match object; span=(8, 1218), match='[\\n  {\\n    \"question\": \"2022年RJ/CRB价格指数变动方差是多少？\">, Expecting value: line 18 column 17 (char 522)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a8af6d4ba31495b8bab5bccb973976d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/28 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "qa_df = build_qa_df(detailed_qa_dict, uuid2doc)\n",
    "large_context_qa_df = build_qa_df(large_context_qa_dict, uuid2large_doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47fbc0b6-4812-402f-93e8-0440e91cb96d",
   "metadata": {
    "id": "47fbc0b6-4812-402f-93e8-0440e91cb96d",
    "outputId": "f98044c7-7260-4930-ae49-39ce3428a4fe"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "360"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(qa_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90824aef-0246-4b54-a833-85362a7820b0",
   "metadata": {
    "id": "90824aef-0246-4b54-a833-85362a7820b0"
   },
   "outputs": [],
   "source": [
    "qa_df.drop_duplicates('question', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8de81918-8c9b-4fc4-ade1-d7c6596f5d13",
   "metadata": {
    "id": "8de81918-8c9b-4fc4-ade1-d7c6596f5d13",
    "outputId": "53354cb2-7486-4c0d-bf6f-2384dedbe61b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "350"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(qa_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35a1502a-0cdd-4eb1-9456-d87fdb6a6c11",
   "metadata": {
    "id": "35a1502a-0cdd-4eb1-9456-d87fdb6a6c11",
    "outputId": "fd8d4d21-b0bc-4976-957d-cfdedc4b1b0e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "56"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(large_context_qa_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bffa7fe-f76b-410b-9014-098ac9c2cc0b",
   "metadata": {
    "id": "6bffa7fe-f76b-410b-9014-098ac9c2cc0b"
   },
   "outputs": [],
   "source": [
    "large_context_qa_df.drop_duplicates('question', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9de97333-1b53-4db3-8463-64256732db22",
   "metadata": {
    "id": "9de97333-1b53-4db3-8463-64256732db22",
    "outputId": "773c4897-daab-4885-994f-04b91d8dee30"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "56"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(large_context_qa_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c22f4e9f-eaff-4303-9db2-18b44f62225f",
   "metadata": {
    "id": "c22f4e9f-eaff-4303-9db2-18b44f62225f",
    "outputId": "b8aa23bb-4535-41ca-93c9-58f972edc4ed"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>context</th>\n",
       "      <th>doc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>9d41e5b3-2a8c-4584-b7b2-a6b7489758c9</td>\n",
       "      <td>土耳其央行在10月加息了多少个基点？</td>\n",
       "      <td>500个基点</td>\n",
       "      <td>土耳其央行10月大幅加息500个基点</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 23 2024年\\n图14：影响国际金融动态的六大因...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>8cf24c61-8169-41e5-8e05-870bf85a4bc8</td>\n",
       "      <td>中国和巴西在2023年有什么货币合作的进展？</td>\n",
       "      <td>中国与巴西签署了在巴西建立人民币清算安排的合作备忘录，并实现了首次人民币贸易结算全流程闭环操作。</td>\n",
       "      <td>3月，中国与巴西签署在巴西建立人民币清算安排的合作备忘录，10月，两国实现首次人民币贸易结算...</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 21 2024年\\n专栏2：全球发展中国家合作进展\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>9870421a-c6c9-4ca2-a394-9053497234e5</td>\n",
       "      <td>服务消费对美国前三季私人消费增长的贡献率是多少？</td>\n",
       "      <td>58.7%</td>\n",
       "      <td>服务支出成为消费者支出领域中最稳定和增幅最大的板块，前三季\\n度服务消费贡献了私人消费增长的...</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 9 2024年\\n综合上述因素，“复苏+分化”将成为...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306</th>\n",
       "      <td>c95e38d5-6741-438f-85e7-7d8f39804b7a</td>\n",
       "      <td>阿美公司的这笔交易金额是多少？</td>\n",
       "      <td>124亿美元</td>\n",
       "      <td>交易金额达124亿美元，</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 43 2024年\\n以促进公私合作伙伴关系和放宽外国...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>3c0da094-8444-428b-9729-f1caa33e4b06</td>\n",
       "      <td>2024年预计新兴经济体的整体经济增速是多少？</td>\n",
       "      <td>4%左右</td>\n",
       "      <td>展望2024年，新兴经济体增长机遇与挑战并存，“南南合作”将成为重要抓手，预计整体经济增速将...</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 19 2024年\\n新兴经济体增长的主要因素，三季度...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     uuid                  question  \\\n",
       "182  9d41e5b3-2a8c-4584-b7b2-a6b7489758c9        土耳其央行在10月加息了多少个基点？   \n",
       "161  8cf24c61-8169-41e5-8e05-870bf85a4bc8    中国和巴西在2023年有什么货币合作的进展？   \n",
       "74   9870421a-c6c9-4ca2-a394-9053497234e5  服务消费对美国前三季私人消费增长的贡献率是多少？   \n",
       "306  c95e38d5-6741-438f-85e7-7d8f39804b7a           阿美公司的这笔交易金额是多少？   \n",
       "150  3c0da094-8444-428b-9729-f1caa33e4b06   2024年预计新兴经济体的整体经济增速是多少？   \n",
       "\n",
       "                                               answer  \\\n",
       "182                                            500个基点   \n",
       "161  中国与巴西签署了在巴西建立人民币清算安排的合作备忘录，并实现了首次人民币贸易结算全流程闭环操作。   \n",
       "74                                              58.7%   \n",
       "306                                            124亿美元   \n",
       "150                                              4%左右   \n",
       "\n",
       "                                               context  \\\n",
       "182                                 土耳其央行10月大幅加息500个基点   \n",
       "161  3月，中国与巴西签署在巴西建立人民币清算安排的合作备忘录，10月，两国实现首次人民币贸易结算...   \n",
       "74   服务支出成为消费者支出领域中最稳定和增幅最大的板块，前三季\\n度服务消费贡献了私人消费增长的...   \n",
       "306                                       交易金额达124亿美元，   \n",
       "150  展望2024年，新兴经济体增长机遇与挑战并存，“南南合作”将成为重要抓手，预计整体经济增速将...   \n",
       "\n",
       "                                                   doc  \n",
       "182  全球经济金融展望报告\\n中国银行研究院 23 2024年\\n图14：影响国际金融动态的六大因...  \n",
       "161  全球经济金融展望报告\\n中国银行研究院 21 2024年\\n专栏2：全球发展中国家合作进展\\...  \n",
       "74   全球经济金融展望报告\\n中国银行研究院 9 2024年\\n综合上述因素，“复苏+分化”将成为...  \n",
       "306  全球经济金融展望报告\\n中国银行研究院 43 2024年\\n以促进公私合作伙伴关系和放宽外国...  \n",
       "150  全球经济金融展望报告\\n中国银行研究院 19 2024年\\n新兴经济体增长的主要因素，三季度...  "
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7700fad3-8829-4545-bb17-b0d023653524",
   "metadata": {
    "id": "7700fad3-8829-4545-bb17-b0d023653524"
   },
   "outputs": [],
   "source": [
    "qa_df['qa_type'] = 'detailed'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85518b60-7120-4b90-95b7-7f3486db7d6c",
   "metadata": {
    "id": "85518b60-7120-4b90-95b7-7f3486db7d6c",
    "outputId": "bb970a44-aa10-46a9-bd4b-8a7fb8d9cf7b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>context</th>\n",
       "      <th>doc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0460392f-a8d7-49e1-8f20-4ef39c163f17</td>\n",
       "      <td>2024年欧洲经济，尤其是欧元区和英国的展望如何？</td>\n",
       "      <td>2024年，欧洲经济增长动能不足，预计疲软表现将延续至上半年。欧元区受到欧洲央行收紧货币政策...</td>\n",
       "      <td></td>\n",
       "      <td>响2024年美国经济走势的重要因素。2023年美国通胀已明显放缓，受基期效\\n应减弱、住宅成...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>10ca1218-3b51-46c3-b8c1-43521a5d22ed</td>\n",
       "      <td>日本可能在何时退出负利率政策，以及预计的利率调整策略是什么？</td>\n",
       "      <td>预计日本可能在2024年初退出负利率政策，将启动首次加息，但加息幅度较小，调整速度会比较缓慢。</td>\n",
       "      <td></td>\n",
       "      <td>动性上升，公司债券和公司信贷市场风险、银行间和货币市场风险攀升。之后ROFCI月均值在不稳定...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>171b54d9-f870-40fe-af92-7d308ee002b6</td>\n",
       "      <td>2024年美国经济可能的走势如何？</td>\n",
       "      <td>预计2024年美国经济增速将进一步回落。存在“软着陆”和衰退两种可能性，但目前“软着陆”的可...</td>\n",
       "      <td></td>\n",
       "      <td>综合上述因素，“复苏+分化”将成为2024年全球经济主线逻辑，预计全\\n年经济增速在2.5%...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>d98dbebc-1683-410e-9c04-97bfad83d7c1</td>\n",
       "      <td>海湾六国在新能源转型方面有哪些具体的举措？</td>\n",
       "      <td>海湾六国中的多个国家设定了净零排放目标，如阿联酋承诺到2050年实现净零排放，沙特、阿曼、科...</td>\n",
       "      <td></td>\n",
       "      <td>供了空间，进一步坚定了海湾六国经济转型的决心。\\n近年来，海湾六国努力推进新能源转型，增强国...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>43ec2dde-d2cb-4cce-bc24-55fd5cb341f5</td>\n",
       "      <td>2023年二季度美国办公楼宇的空置率是多少？</td>\n",
       "      <td>2023年二季度美国办公楼宇的空置率为18.2%。</td>\n",
       "      <td></td>\n",
       "      <td>第七，美联储加息以来住房贷款质量稳定。美国单一家庭房贷整体拖欠率\\n季度平均值从2.1%下降...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    uuid                        question  \\\n",
       "11  0460392f-a8d7-49e1-8f20-4ef39c163f17       2024年欧洲经济，尤其是欧元区和英国的展望如何？   \n",
       "29  10ca1218-3b51-46c3-b8c1-43521a5d22ed  日本可能在何时退出负利率政策，以及预计的利率调整策略是什么？   \n",
       "9   171b54d9-f870-40fe-af92-7d308ee002b6               2024年美国经济可能的走势如何？   \n",
       "44  d98dbebc-1683-410e-9c04-97bfad83d7c1           海湾六国在新能源转型方面有哪些具体的举措？   \n",
       "52  43ec2dde-d2cb-4cce-bc24-55fd5cb341f5          2023年二季度美国办公楼宇的空置率是多少？   \n",
       "\n",
       "                                               answer context  \\\n",
       "11  2024年，欧洲经济增长动能不足，预计疲软表现将延续至上半年。欧元区受到欧洲央行收紧货币政策...           \n",
       "29    预计日本可能在2024年初退出负利率政策，将启动首次加息，但加息幅度较小，调整速度会比较缓慢。           \n",
       "9   预计2024年美国经济增速将进一步回落。存在“软着陆”和衰退两种可能性，但目前“软着陆”的可...           \n",
       "44  海湾六国中的多个国家设定了净零排放目标，如阿联酋承诺到2050年实现净零排放，沙特、阿曼、科...           \n",
       "52                          2023年二季度美国办公楼宇的空置率为18.2%。           \n",
       "\n",
       "                                                  doc  \n",
       "11  响2024年美国经济走势的重要因素。2023年美国通胀已明显放缓，受基期效\\n应减弱、住宅成...  \n",
       "29  动性上升，公司债券和公司信贷市场风险、银行间和货币市场风险攀升。之后ROFCI月均值在不稳定...  \n",
       "9   综合上述因素，“复苏+分化”将成为2024年全球经济主线逻辑，预计全\\n年经济增速在2.5%...  \n",
       "44  供了空间，进一步坚定了海湾六国经济转型的决心。\\n近年来，海湾六国努力推进新能源转型，增强国...  \n",
       "52  第七，美联储加息以来住房贷款质量稳定。美国单一家庭房贷整体拖欠率\\n季度平均值从2.1%下降...  "
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "large_context_qa_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95e6b451-5603-480f-88b7-21d61598c65d",
   "metadata": {
    "id": "95e6b451-5603-480f-88b7-21d61598c65d"
   },
   "outputs": [],
   "source": [
    "large_context_qa_df['qa_type'] = 'large_context'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f056cd2-2940-4e46-99d7-9985d3abdb52",
   "metadata": {
    "id": "2f056cd2-2940-4e46-99d7-9985d3abdb52"
   },
   "outputs": [],
   "source": [
    "qa_df = pd.concat([qa_df, large_context_qa_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6040ba8-1d6d-4c61-aa6a-772b787713d9",
   "metadata": {
    "id": "d6040ba8-1d6d-4c61-aa6a-772b787713d9",
    "outputId": "c079ceb8-2031-41dd-b9d5-57aa6451a51d"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>context</th>\n",
       "      <th>doc</th>\n",
       "      <th>qa_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>61c25726-912a-4bd4-981c-01e9cfa81951</td>\n",
       "      <td>面对经济增长放缓和全球经济挑战，海湾国家采取了什么措施？</td>\n",
       "      <td>为了应对经济增长放缓和全球经济挑战，沙特、阿联酋等国政府大力推动绿色经济转型和经济多元化改革...</td>\n",
       "      <td></td>\n",
       "      <td>注重绿色经济发展，为中海经贸合作创造了良好机遇。\\n（一）近年来海湾六国经济表现受能源价格影...</td>\n",
       "      <td>large_context</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>b78e18df-81e8-44d6-a5ed-800fda878b7d</td>\n",
       "      <td>欧盟在认定补贴属于可诉性补贴后，可能会采取哪些措施？</td>\n",
       "      <td>欧盟在认定补贴属于可诉性补贴后，一般可能采取加征反补贴税和要求出口商接受价格承诺这两种措施。...</td>\n",
       "      <td></td>\n",
       "      <td>存在专向性1或属于禁止性补贴，且对欧盟内部产业构成实质性损害的情况下，欧盟即可认定补贴属于\\...</td>\n",
       "      <td>large_context</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287</th>\n",
       "      <td>28c7d4f0-8cea-48b3-9cfa-6683446fd425</td>\n",
       "      <td>海湾六国经济结构的主要问题是什么？</td>\n",
       "      <td>经济结构相对单一</td>\n",
       "      <td>鉴于海湾六国经济结构相对单一，非石油部门虽然为海</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 40 2024年\\n注重绿色经济发展，为中海经贸合作...</td>\n",
       "      <td>detailed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>a9756922-8c03-4a1c-a9f9-02f65b61068d</td>\n",
       "      <td>海湾六国为什么在经济发展上有别于其他中东国家?</td>\n",
       "      <td>海湾六国因为拥有丰富的油气资源，这使得它们在经济发展上积累了大量国民财富，同时，它们的对外开...</td>\n",
       "      <td></td>\n",
       "      <td>专栏4：大宗商品价格波动与全球金融系统稳定\\n近年来，全球大宗商品市场波动加剧。从历年大宗商...</td>\n",
       "      <td>large_context</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>320</th>\n",
       "      <td>873ee876-8549-49b9-b182-6584113b2701</td>\n",
       "      <td>中国与海湾国家的经贸合作可能如何影响人民币国际化？</td>\n",
       "      <td>通过加强双边经贸合作，人民币跨境贸易支付可能会成为人民币在中东地区使用的主要切入点，进而扩展...</td>\n",
       "      <td>当前，中国与海湾六国拥有紧密的经贸联系，并且随着19个阿拉伯国家相继加入“一带一路”倡议，人...</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 45 2024年\\n第四，中海深化经贸合作将为人民币...</td>\n",
       "      <td>detailed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     uuid                      question  \\\n",
       "43   61c25726-912a-4bd4-981c-01e9cfa81951  面对经济增长放缓和全球经济挑战，海湾国家采取了什么措施？   \n",
       "16   b78e18df-81e8-44d6-a5ed-800fda878b7d    欧盟在认定补贴属于可诉性补贴后，可能会采取哪些措施？   \n",
       "287  28c7d4f0-8cea-48b3-9cfa-6683446fd425             海湾六国经济结构的主要问题是什么？   \n",
       "41   a9756922-8c03-4a1c-a9f9-02f65b61068d       海湾六国为什么在经济发展上有别于其他中东国家?   \n",
       "320  873ee876-8549-49b9-b182-6584113b2701     中国与海湾国家的经贸合作可能如何影响人民币国际化？   \n",
       "\n",
       "                                                answer  \\\n",
       "43   为了应对经济增长放缓和全球经济挑战，沙特、阿联酋等国政府大力推动绿色经济转型和经济多元化改革...   \n",
       "16   欧盟在认定补贴属于可诉性补贴后，一般可能采取加征反补贴税和要求出口商接受价格承诺这两种措施。...   \n",
       "287                                           经济结构相对单一   \n",
       "41   海湾六国因为拥有丰富的油气资源，这使得它们在经济发展上积累了大量国民财富，同时，它们的对外开...   \n",
       "320  通过加强双边经贸合作，人民币跨境贸易支付可能会成为人民币在中东地区使用的主要切入点，进而扩展...   \n",
       "\n",
       "                                               context  \\\n",
       "43                                                       \n",
       "16                                                       \n",
       "287                           鉴于海湾六国经济结构相对单一，非石油部门虽然为海   \n",
       "41                                                       \n",
       "320  当前，中国与海湾六国拥有紧密的经贸联系，并且随着19个阿拉伯国家相继加入“一带一路”倡议，人...   \n",
       "\n",
       "                                                   doc        qa_type  \n",
       "43   注重绿色经济发展，为中海经贸合作创造了良好机遇。\\n（一）近年来海湾六国经济表现受能源价格影...  large_context  \n",
       "16   存在专向性1或属于禁止性补贴，且对欧盟内部产业构成实质性损害的情况下，欧盟即可认定补贴属于\\...  large_context  \n",
       "287  全球经济金融展望报告\\n中国银行研究院 40 2024年\\n注重绿色经济发展，为中海经贸合作...       detailed  \n",
       "41   专栏4：大宗商品价格波动与全球金融系统稳定\\n近年来，全球大宗商品市场波动加剧。从历年大宗商...  large_context  \n",
       "320  全球经济金融展望报告\\n中国银行研究院 45 2024年\\n第四，中海深化经贸合作将为人民币...       detailed  "
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e759a73-28fd-485c-bb28-4a55f804584a",
   "metadata": {
    "id": "2e759a73-28fd-485c-bb28-4a55f804584a",
    "outputId": "a8444232-9502-44b1-bfb1-843b8f975b30"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "406"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(qa_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8633f77f-0f13-42d1-ac52-a5a20a794db3",
   "metadata": {
    "id": "8633f77f-0f13-42d1-ac52-a5a20a794db3"
   },
   "source": [
    "# QA质量检查"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5331d71-3f37-4532-9b26-f5bb653804aa",
   "metadata": {
    "id": "a5331d71-3f37-4532-9b26-f5bb653804aa"
   },
   "outputs": [],
   "source": [
    "# qa_check_prompt_tmpl = \"\"\"\n",
    "# 你是一个优秀的NLP方面的助教，你的任务是帮助教授检查期末考试的阅读理解题。\n",
    "# 你需要根据上下文（<context></context>之间的部分），所出的问题（<question></question>之间的部分），以及答案（<answer></answer>）进行打分。\n",
    "# 打分需要检查如下3点，并分别对这3点进行打分：\n",
    "# 1. answer_relevance: 答案相关性，答案是否与问题相关，相关为1，不相关为0\n",
    "# 2. context_relevance: 上下文相关性，问题和上下文是否相关，相关为1，不相关为0\n",
    "# 3. groundness: 真实性，检查答案是否是基于上下文回答的，是为1，否为0\n",
    "# 结果请以JSON形式组织，格式为如下：\n",
    "# {\"answer_relevance\": ..., \"context_relevance\": ..., \"groundness\": ...}\n",
    "\n",
    "# 上下文：\n",
    "# <context>\n",
    "# {{context}}\n",
    "# </context>\n",
    "\n",
    "# 问题：\n",
    "# <question>\n",
    "# {{question}}\n",
    "# </question>\n",
    "\n",
    "# 答案：\n",
    "# <answer>\n",
    "# {{answer}}\n",
    "# </answer>\n",
    "\n",
    "# 请打分：\n",
    "# \"\"\"\n",
    "\n",
    "qa_check_prompt_tmpl = \"\"\"\n",
    "你是一个优秀的NLP方面的助教，你的任务是帮助检查出题组所出的期末考试的题目。\n",
    "你需要根据所出的问题（<question></question>之间的部分），以及参考答案（<answer></answer>）进行打分，并给出打分理由，分值是一个int类型的值，取值范围为1-5。\n",
    "好的问题，应该是询问实时、观点等，而不是类似于“这一段描述了什么”，“文本描述了什么”；\n",
    "好的答案，应该能够直接回答问题，而不是给出在原文中的引用，例如“第3章”等\n",
    "\n",
    "结果请以JSON形式组织，格式为如下：\n",
    "{\"score\": ..., \"reason\": ...}\n",
    "\n",
    "问题：\n",
    "<question>\n",
    "{{question}}\n",
    "</question>\n",
    "\n",
    "答案：\n",
    "<answer>\n",
    "{{answer}}\n",
    "</answer>\n",
    "\n",
    "请打分：\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "219411e2-08b9-4232-9a18-67d4a1147fae",
   "metadata": {
    "id": "219411e2-08b9-4232-9a18-67d4a1147fae",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def build_qa_scoring_prompt(row):\n",
    "    question = row['question']\n",
    "    answer = row['answer']\n",
    "    prompt = qa_check_prompt_tmpl.replace('{{question}}', question).replace('{{answer}}', answer)\n",
    "\n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "feb8207c-8bd1-439f-be8d-46e9f02cbc6a",
   "metadata": {
    "colab": {
     "referenced_widgets": [
      "3deeb865a0a84a89a614ff4355ef6212"
     ]
    },
    "id": "feb8207c-8bd1-439f-be8d-46e9f02cbc6a",
    "outputId": "cad81b32-0cb1-45c0-b68c-ce592550d373"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3deeb865a0a84a89a614ff4355ef6212",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/405 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import threading\n",
    "import concurrent.futures\n",
    "from tqdm.auto import tqdm\n",
    "import json\n",
    "\n",
    "qa_scoring_ckpt = {}\n",
    "qa_scoring_ckpt_filename = os.path.join(output_dir, f\"qa_scoring_ckpt.jsonl\")\n",
    "if os.path.exists(qa_scoring_ckpt_filename):\n",
    "    qa_scoring_ckpt = open(qa_scoring_ckpt_filename).readlines()\n",
    "    qa_scoring_ckpt = [json.loads(line.strip()) for line in qa_scoring_ckpt if line.strip() != '']\n",
    "    qa_scoring_ckpt = {item['question']: item for item in qa_scoring_ckpt}\n",
    "    print(f'found checkpoint, item count: {len(qa_scoring_ckpt)}')\n",
    "\n",
    "file_lock = threading.Lock()\n",
    "max_workers = 2\n",
    "with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:\n",
    "    futures = {row['question']: executor.submit(chat, build_qa_scoring_prompt(row), 3, True) for _, row in qa_df.iterrows() if row['question'] not in qa_scoring_ckpt}\n",
    "    for question in tqdm(futures):\n",
    "        future = futures[question]\n",
    "        result = future.result()\n",
    "        if result is None:\n",
    "            continue\n",
    "\n",
    "        item = {'question': question, 'raw_resp': result}\n",
    "        qa_scoring_ckpt[question] = item\n",
    "\n",
    "        global file_lock\n",
    "        file_lock.acquire()\n",
    "\n",
    "        try:\n",
    "            with open(qa_scoring_ckpt_filename, 'a') as f:\n",
    "                f.write(json.dumps(item, ensure_ascii=False) + '\\n')\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "        finally:\n",
    "            file_lock.release()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c49ef2a7-cede-4199-9be4-9069a33eaa19",
   "metadata": {
    "id": "c49ef2a7-cede-4199-9be4-9069a33eaa19"
   },
   "outputs": [],
   "source": [
    "qa_scoring_dict = {}\n",
    "for key, value in qa_scoring_ckpt.items():\n",
    "    try:\n",
    "        qa_scoring_dict[key] = json.loads(value['raw_resp'].replace('”}', '\"}'))\n",
    "        if 'score' not in qa_scoring_dict[key]:\n",
    "            raise ValueError(f'no score in result, question: {key}')\n",
    "    except Exception as e:\n",
    "        print(f\"{key}, error: {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1a6058d-4575-4f14-b8f0-12c788807f78",
   "metadata": {
    "id": "a1a6058d-4575-4f14-b8f0-12c788807f78"
   },
   "outputs": [],
   "source": [
    "qa_df['score'] = qa_df['question'].apply(lambda q: qa_scoring_dict[q]['score'])\n",
    "qa_df['score_reason'] = qa_df['question'].apply(lambda q: qa_scoring_dict[q]['reason'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c276f215-0dff-4805-a325-7fad2b4f7cc9",
   "metadata": {
    "id": "c276f215-0dff-4805-a325-7fad2b4f7cc9",
    "outputId": "2014910c-3d49-4582-cc1d-cbcf77f3e2cf"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>context</th>\n",
       "      <th>doc</th>\n",
       "      <th>qa_type</th>\n",
       "      <th>score</th>\n",
       "      <th>score_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>a52f60d5-4430-405f-a2dd-8a975ac03b91</td>\n",
       "      <td>ROFCI在哪一区域表明金融市场不稳定？</td>\n",
       "      <td>不稳定区域</td>\n",
       "      <td>仍然处于不稳定区域，但是月度指标及季度平均值在全年呈波动下降趋势。</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 25 2024年\\n图15：美联储资产规模变化（亿美...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>4</td>\n",
       "      <td>问题询问的是具体区域，答案直接给出了相应答案，但'不稳定区域'可能需要更多的上下文信息来精确指代。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343</th>\n",
       "      <td>c47057d6-35c0-4466-bafd-4ed036030c1d</td>\n",
       "      <td>在2024年有多少比例的办公空间租赁合同将到期？</td>\n",
       "      <td>17%</td>\n",
       "      <td>预计美国17%的办公空间租赁合同将在2024年底前到期</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 47 2024年\\n第七，美联储加息以来住房贷款质量...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>5</td>\n",
       "      <td>问题询问的是具体数据，具有实现实时查询的价值；答案直接给出了一个明确的比例，无需额外查找信息...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>展望2024年，全球经济复苏的预期如何？</td>\n",
       "      <td>全球经济复苏预计将依旧疲软。</td>\n",
       "      <td>展望2024年，预计全球经济复苏将依旧疲软</td>\n",
       "      <td>研究院\\n全球经济金融展望报告\\n要点2024年年报（总第57期） 报告日期：2023年12...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>5</td>\n",
       "      <td>问题询问的是未来的经济预期，属于观点和预测的范畴，符合优质问题的标准。答案直接给出了对经济复...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>a52f60d5-4430-405f-a2dd-8a975ac03b91</td>\n",
       "      <td>2023年1-11月，美国金融危机风险指标的平均值是多少？</td>\n",
       "      <td>42.33</td>\n",
       "      <td>2023年1-11月，美国金融危机风险指标（ROFCI）的月度平均值从2022年的42.83...</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 25 2024年\\n图15：美联储资产规模变化（亿美...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>5</td>\n",
       "      <td>问题询问的是具体的数据，需要考生对实时信息有了解，答案直接给出了数值，符合要求。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>56a6c642-9f39-413e-93dd-362b30d9fa5b</td>\n",
       "      <td>2024年全球股票市场有哪些主要特征？</td>\n",
       "      <td>2024年全球股票市场的主要特征包括：美国股票市场受到美国通胀回落、美联储暂停加息及企业盈利...</td>\n",
       "      <td></td>\n",
       "      <td>图19：美国联邦基金目标利率与全球MSCI指数\\n资料来源：Wind，中国银行研究院\\n表3...</td>\n",
       "      <td>large_context</td>\n",
       "      <td>5</td>\n",
       "      <td>问题询问的是未来全球股票市场的特征，要求有分析和预测，答案提供了具体的特征，并且给出了支持这...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     uuid                       question  \\\n",
       "197  a52f60d5-4430-405f-a2dd-8a975ac03b91           ROFCI在哪一区域表明金融市场不稳定？   \n",
       "343  c47057d6-35c0-4466-bafd-4ed036030c1d       在2024年有多少比例的办公空间租赁合同将到期？   \n",
       "5    e73a0c9d-d42b-4350-a4c3-b38bf67c68a5           展望2024年，全球经济复苏的预期如何？   \n",
       "196  a52f60d5-4430-405f-a2dd-8a975ac03b91  2023年1-11月，美国金融危机风险指标的平均值是多少？   \n",
       "34   56a6c642-9f39-413e-93dd-362b30d9fa5b            2024年全球股票市场有哪些主要特征？   \n",
       "\n",
       "                                                answer  \\\n",
       "197                                              不稳定区域   \n",
       "343                                                17%   \n",
       "5                                       全球经济复苏预计将依旧疲软。   \n",
       "196                                              42.33   \n",
       "34   2024年全球股票市场的主要特征包括：美国股票市场受到美国通胀回落、美联储暂停加息及企业盈利...   \n",
       "\n",
       "                                               context  \\\n",
       "197                  仍然处于不稳定区域，但是月度指标及季度平均值在全年呈波动下降趋势。   \n",
       "343                        预计美国17%的办公空间租赁合同将在2024年底前到期   \n",
       "5                                展望2024年，预计全球经济复苏将依旧疲软   \n",
       "196  2023年1-11月，美国金融危机风险指标（ROFCI）的月度平均值从2022年的42.83...   \n",
       "34                                                       \n",
       "\n",
       "                                                   doc        qa_type  score  \\\n",
       "197  全球经济金融展望报告\\n中国银行研究院 25 2024年\\n图15：美联储资产规模变化（亿美...       detailed      4   \n",
       "343  全球经济金融展望报告\\n中国银行研究院 47 2024年\\n第七，美联储加息以来住房贷款质量...       detailed      5   \n",
       "5    研究院\\n全球经济金融展望报告\\n要点2024年年报（总第57期） 报告日期：2023年12...       detailed      5   \n",
       "196  全球经济金融展望报告\\n中国银行研究院 25 2024年\\n图15：美联储资产规模变化（亿美...       detailed      5   \n",
       "34   图19：美国联邦基金目标利率与全球MSCI指数\\n资料来源：Wind，中国银行研究院\\n表3...  large_context      5   \n",
       "\n",
       "                                          score_reason  \n",
       "197  问题询问的是具体区域，答案直接给出了相应答案，但'不稳定区域'可能需要更多的上下文信息来精确指代。  \n",
       "343  问题询问的是具体数据，具有实现实时查询的价值；答案直接给出了一个明确的比例，无需额外查找信息...  \n",
       "5    问题询问的是未来的经济预期，属于观点和预测的范畴，符合优质问题的标准。答案直接给出了对经济复...  \n",
       "196           问题询问的是具体的数据，需要考生对实时信息有了解，答案直接给出了数值，符合要求。  \n",
       "34   问题询问的是未来全球股票市场的特征，要求有分析和预测，答案提供了具体的特征，并且给出了支持这...  "
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cf12c03-48c4-49ca-b48c-d727cd9ff12f",
   "metadata": {
    "id": "0cf12c03-48c4-49ca-b48c-d727cd9ff12f",
    "outputId": "2d84d359-9310-4c82-9f6f-01c436fd285a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "score\n",
       "5    326\n",
       "4     64\n",
       "3     15\n",
       "2      1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa_df['score'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aa437c6-8ef5-4972-b7f7-0f5b18263541",
   "metadata": {
    "id": "4aa437c6-8ef5-4972-b7f7-0f5b18263541",
    "outputId": "b47b0480-341c-4771-8ce9-3901030fd63b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>context</th>\n",
       "      <th>doc</th>\n",
       "      <th>qa_type</th>\n",
       "      <th>score</th>\n",
       "      <th>score_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1b782370-c7f8-48a8-9647-591bb91f27f7</td>\n",
       "      <td>报告中提到的主要经济体GDP增速变化趋势的图的名称是什么？</td>\n",
       "      <td>主要经济体GDP增速变化趋势</td>\n",
       "      <td>图2：主要经济体GDP增速变化趋势（%）</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 3 2024年\\n图2：主要经济体GDP增速变化趋势...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题要求回答图表的名称，而答案直接复述了问题中的内容，没有提供具体的信息。一个好的答案应当是...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>ded827f2-f0de-429b-9912-344018765257</td>\n",
       "      <td>图3展示了什么内容的变化趋势？</td>\n",
       "      <td>部分经济体工业生产指数</td>\n",
       "      <td>图3：部分经济体工业生产指数变化趋势（2010年=100）</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 4 2024年\\n图3：部分经济体工业生产指数变化趋...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题要求描述图表内容的变化趋势，但没有询问具体的细节或需要分析。答案直接给出了图表的主题，没...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>4bd96918-ce66-4762-a725-8a7e4b543dbe</td>\n",
       "      <td>图4展示了什么数据？</td>\n",
       "      <td>部分欧洲国家的零售销售指数</td>\n",
       "      <td>图4：部分欧洲国家零售销售指数</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 5 2024年\\n图4：部分欧洲国家零售销售指数\\n...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题要求回答图4展示的具体内容，答案直接提供了信息，但没有提及是否是根据图4得出的结论，没有...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>2e253555-a34a-48cb-824b-da74f33a9111</td>\n",
       "      <td>图6中的数据来源是哪里？</td>\n",
       "      <td>IMF，中国银行研究院</td>\n",
       "      <td>图6：全球投资率变化趋势（%） 资料来源：IMF，中国银行研究院</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 6 2024年\\n2023年全球跨境直接投资将继续下...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题要求回答数据来源，但没有提供足够的上下文使得答案能在不查看图6的情况下被理解。同时，答案...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>774ab320-aba8-45a7-a5f4-791782b46d08</td>\n",
       "      <td>哪些因素将对经济形成托底，减缓经济下行幅度？</td>\n",
       "      <td>一些稳定因素</td>\n",
       "      <td>与此同时，一些稳定因素将逐渐显现，对经济形成托底，减缓经济下行幅度。</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 8 2024年\\n率政策。\\n二是各国经济走势将进一...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题询问的是经济的托底因素，具有开放性和探讨性，是一个不错的问题。但答案'一些稳定因素'过于...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>733c7f0b-06fd-40f4-a1de-0c863db6069a</td>\n",
       "      <td>哪些因素可能支撑亚太新兴经济体的内需？</td>\n",
       "      <td>信息不足，无法直接回答，需要更多上下文信息。</td>\n",
       "      <td>不过强劲的内需是支撑亚太</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 18 2024年\\n图12：日本通胀同比增速变化（%...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题询问的是支撑亚太新兴经济体内需的因素，这是一个开放性问题，可以探讨多种可能性，但答案表示...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>883b0471-5082-44f4-b625-86386102bc2d</td>\n",
       "      <td>图13展示了什么内容？</td>\n",
       "      <td>部分亚洲新兴经济体GDP同比增速</td>\n",
       "      <td>图13：部分亚洲新兴经济体GDP同比增速（%）</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 20 2024年\\n图13：部分亚洲新兴经济体GDP...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题要求描述图片内容，但更好的问题是引导学生分析或解释图表的意义，而不仅仅是复述。答案直接回...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>9d41e5b3-2a8c-4584-b7b2-a6b7489758c9</td>\n",
       "      <td>俄罗斯央行在何时加息了200个基点？</td>\n",
       "      <td>未在文中给出具体时间</td>\n",
       "      <td>俄罗斯央行加息200个基点</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 23 2024年\\n图14：影响国际金融动态的六大因...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题询问的是具体事件的时间，属于事实性问题，但答案指出信息未提供，说明题目可能缺乏适当的上下...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>a52f60d5-4430-405f-a2dd-8a975ac03b91</td>\n",
       "      <td>图15展示的是什么内容？</td>\n",
       "      <td>美联储资产规模变化</td>\n",
       "      <td>图15：美联储资产规模变化（亿美元）\\n资料来源：美联储，中国银行研究院</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 25 2024年\\n图15：美联储资产规模变化（亿美...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题要求描述图片内容，但优秀的问题应更倾向于询问实时信息或观点。答案直接回答了问题，但没有提...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225</th>\n",
       "      <td>65b167b2-bf81-4148-b6a6-7afb9946676d</td>\n",
       "      <td>报告讨论的时间范围是什么？</td>\n",
       "      <td>2024年</td>\n",
       "      <td>全球经济金融展望报告 中国银行研究院 30 2024年</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 30 2024年\\n图18：主要货币兑美元汇率较20...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题询问的是时间范围，但答案只给出了一个具体的年份，没有明确是否是开始或结束时间，也没有提供...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>65b167b2-bf81-4148-b6a6-7afb9946676d</td>\n",
       "      <td>图18展示了什么数据变化？</td>\n",
       "      <td>主要货币兑美元汇率的变化</td>\n",
       "      <td>图18：主要货币兑美元汇率较2023年年初变动（%）</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 30 2024年\\n图18：主要货币兑美元汇率较20...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题描述不够具体，没有提供足够的信息让考生无需查阅图18就能理解问题。答案直接回答了问题，但...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258</th>\n",
       "      <td>d58699ac-79da-4321-98c5-56c0350d2ba5</td>\n",
       "      <td>2024年，哪些行业的债券融资功能可能增强？</td>\n",
       "      <td>日元和人民币的债券融资功能</td>\n",
       "      <td>部分低利率货币的债券融资功能有望增强。日元和人民币在国际金融市场的融资功能明显上升。</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 35 2024年\\n体国债收益率出现较大幅度下跌。当...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题询问的是未来的行业趋势，这需要考生对金融市场的深入理解和预测，属于较好的开放性问题。但是...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297</th>\n",
       "      <td>c6fc6079-0daf-4d6a-bd6e-7d9820012425</td>\n",
       "      <td>哪个国家的可再生能源装机量在2022年超过了过去十年的总和？</td>\n",
       "      <td>卡塔尔和阿曼</td>\n",
       "      <td>2022年卡塔尔和阿曼的可再生能源装机量超过这些国家过去十年的总和。</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 42 2024年\\n供了空间，进一步坚定了海湾六国经...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题询问的是一个具体国家的特定事件，涉及实时信息，符合好的问题标准。但答案给出了两个国家，且...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317</th>\n",
       "      <td>8bd98bdb-bbf3-443f-a671-3c13416500f4</td>\n",
       "      <td>2022年中国-海湾阿拉伯国家合作委员会峰会在哪个国家举行?</td>\n",
       "      <td>文中未提及</td>\n",
       "      <td>2022年12月，习近平主席参加中国-海湾阿拉伯国家合作委员会峰会时提出，中国与海湾国家要“...</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 44 2024年\\n二、三位。\\n（三）未来中海合作...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题询问的是具体事件，属于实时信息的范畴，是好的问题。但答案没有提供实际信息，只是说明了文中...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>8bd98bdb-bbf3-443f-a671-3c13416500f4</td>\n",
       "      <td>海湾六国在全球石油出口中的排名如何?</td>\n",
       "      <td>全球位居前列</td>\n",
       "      <td>而沙特是全球第二大产油国和第一大石油出口国，其他海合会国家的油气产量和出口也在全球位居前列。</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 44 2024年\\n二、三位。\\n（三）未来中海合作...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>3</td>\n",
       "      <td>问题询问的是具体排名，但答案没有提供明确的顺序或位置，只是模糊地说明位居前列。好的答案应该给...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     uuid                        question  \\\n",
       "25   1b782370-c7f8-48a8-9647-591bb91f27f7   报告中提到的主要经济体GDP增速变化趋势的图的名称是什么？   \n",
       "33   ded827f2-f0de-429b-9912-344018765257                 图3展示了什么内容的变化趋势？   \n",
       "40   4bd96918-ce66-4762-a725-8a7e4b543dbe                      图4展示了什么数据？   \n",
       "50   2e253555-a34a-48cb-824b-da74f33a9111                    图6中的数据来源是哪里？   \n",
       "70   774ab320-aba8-45a7-a5f4-791782b46d08          哪些因素将对经济形成托底，减缓经济下行幅度？   \n",
       "143  733c7f0b-06fd-40f4-a1de-0c863db6069a             哪些因素可能支撑亚太新兴经济体的内需？   \n",
       "153  883b0471-5082-44f4-b625-86386102bc2d                     图13展示了什么内容？   \n",
       "183  9d41e5b3-2a8c-4584-b7b2-a6b7489758c9              俄罗斯央行在何时加息了200个基点？   \n",
       "194  a52f60d5-4430-405f-a2dd-8a975ac03b91                    图15展示的是什么内容？   \n",
       "225  65b167b2-bf81-4148-b6a6-7afb9946676d                   报告讨论的时间范围是什么？   \n",
       "226  65b167b2-bf81-4148-b6a6-7afb9946676d                   图18展示了什么数据变化？   \n",
       "258  d58699ac-79da-4321-98c5-56c0350d2ba5          2024年，哪些行业的债券融资功能可能增强？   \n",
       "297  c6fc6079-0daf-4d6a-bd6e-7d9820012425  哪个国家的可再生能源装机量在2022年超过了过去十年的总和？   \n",
       "317  8bd98bdb-bbf3-443f-a671-3c13416500f4  2022年中国-海湾阿拉伯国家合作委员会峰会在哪个国家举行?   \n",
       "318  8bd98bdb-bbf3-443f-a671-3c13416500f4              海湾六国在全球石油出口中的排名如何?   \n",
       "\n",
       "                     answer  \\\n",
       "25           主要经济体GDP增速变化趋势   \n",
       "33              部分经济体工业生产指数   \n",
       "40            部分欧洲国家的零售销售指数   \n",
       "50              IMF，中国银行研究院   \n",
       "70                   一些稳定因素   \n",
       "143  信息不足，无法直接回答，需要更多上下文信息。   \n",
       "153        部分亚洲新兴经济体GDP同比增速   \n",
       "183              未在文中给出具体时间   \n",
       "194               美联储资产规模变化   \n",
       "225                   2024年   \n",
       "226            主要货币兑美元汇率的变化   \n",
       "258           日元和人民币的债券融资功能   \n",
       "297                  卡塔尔和阿曼   \n",
       "317                   文中未提及   \n",
       "318                  全球位居前列   \n",
       "\n",
       "                                               context  \\\n",
       "25                                图2：主要经济体GDP增速变化趋势（%）   \n",
       "33                       图3：部分经济体工业生产指数变化趋势（2010年=100）   \n",
       "40                                     图4：部分欧洲国家零售销售指数   \n",
       "50                    图6：全球投资率变化趋势（%） 资料来源：IMF，中国银行研究院   \n",
       "70                  与此同时，一些稳定因素将逐渐显现，对经济形成托底，减缓经济下行幅度。   \n",
       "143                                       不过强劲的内需是支撑亚太   \n",
       "153                            图13：部分亚洲新兴经济体GDP同比增速（%）   \n",
       "183                                      俄罗斯央行加息200个基点   \n",
       "194               图15：美联储资产规模变化（亿美元）\\n资料来源：美联储，中国银行研究院   \n",
       "225                        全球经济金融展望报告 中国银行研究院 30 2024年   \n",
       "226                         图18：主要货币兑美元汇率较2023年年初变动（%）   \n",
       "258         部分低利率货币的债券融资功能有望增强。日元和人民币在国际金融市场的融资功能明显上升。   \n",
       "297                 2022年卡塔尔和阿曼的可再生能源装机量超过这些国家过去十年的总和。   \n",
       "317  2022年12月，习近平主席参加中国-海湾阿拉伯国家合作委员会峰会时提出，中国与海湾国家要“...   \n",
       "318     而沙特是全球第二大产油国和第一大石油出口国，其他海合会国家的油气产量和出口也在全球位居前列。   \n",
       "\n",
       "                                                   doc   qa_type  score  \\\n",
       "25   全球经济金融展望报告\\n中国银行研究院 3 2024年\\n图2：主要经济体GDP增速变化趋势...  detailed      3   \n",
       "33   全球经济金融展望报告\\n中国银行研究院 4 2024年\\n图3：部分经济体工业生产指数变化趋...  detailed      3   \n",
       "40   全球经济金融展望报告\\n中国银行研究院 5 2024年\\n图4：部分欧洲国家零售销售指数\\n...  detailed      3   \n",
       "50   全球经济金融展望报告\\n中国银行研究院 6 2024年\\n2023年全球跨境直接投资将继续下...  detailed      3   \n",
       "70   全球经济金融展望报告\\n中国银行研究院 8 2024年\\n率政策。\\n二是各国经济走势将进一...  detailed      3   \n",
       "143  全球经济金融展望报告\\n中国银行研究院 18 2024年\\n图12：日本通胀同比增速变化（%...  detailed      3   \n",
       "153  全球经济金融展望报告\\n中国银行研究院 20 2024年\\n图13：部分亚洲新兴经济体GDP...  detailed      3   \n",
       "183  全球经济金融展望报告\\n中国银行研究院 23 2024年\\n图14：影响国际金融动态的六大因...  detailed      3   \n",
       "194  全球经济金融展望报告\\n中国银行研究院 25 2024年\\n图15：美联储资产规模变化（亿美...  detailed      3   \n",
       "225  全球经济金融展望报告\\n中国银行研究院 30 2024年\\n图18：主要货币兑美元汇率较20...  detailed      3   \n",
       "226  全球经济金融展望报告\\n中国银行研究院 30 2024年\\n图18：主要货币兑美元汇率较20...  detailed      3   \n",
       "258  全球经济金融展望报告\\n中国银行研究院 35 2024年\\n体国债收益率出现较大幅度下跌。当...  detailed      3   \n",
       "297  全球经济金融展望报告\\n中国银行研究院 42 2024年\\n供了空间，进一步坚定了海湾六国经...  detailed      3   \n",
       "317  全球经济金融展望报告\\n中国银行研究院 44 2024年\\n二、三位。\\n（三）未来中海合作...  detailed      3   \n",
       "318  全球经济金融展望报告\\n中国银行研究院 44 2024年\\n二、三位。\\n（三）未来中海合作...  detailed      3   \n",
       "\n",
       "                                          score_reason  \n",
       "25   问题要求回答图表的名称，而答案直接复述了问题中的内容，没有提供具体的信息。一个好的答案应当是...  \n",
       "33   问题要求描述图表内容的变化趋势，但没有询问具体的细节或需要分析。答案直接给出了图表的主题，没...  \n",
       "40   问题要求回答图4展示的具体内容，答案直接提供了信息，但没有提及是否是根据图4得出的结论，没有...  \n",
       "50   问题要求回答数据来源，但没有提供足够的上下文使得答案能在不查看图6的情况下被理解。同时，答案...  \n",
       "70   问题询问的是经济的托底因素，具有开放性和探讨性，是一个不错的问题。但答案'一些稳定因素'过于...  \n",
       "143  问题询问的是支撑亚太新兴经济体内需的因素，这是一个开放性问题，可以探讨多种可能性，但答案表示...  \n",
       "153  问题要求描述图片内容，但更好的问题是引导学生分析或解释图表的意义，而不仅仅是复述。答案直接回...  \n",
       "183  问题询问的是具体事件的时间，属于事实性问题，但答案指出信息未提供，说明题目可能缺乏适当的上下...  \n",
       "194  问题要求描述图片内容，但优秀的问题应更倾向于询问实时信息或观点。答案直接回答了问题，但没有提...  \n",
       "225  问题询问的是时间范围，但答案只给出了一个具体的年份，没有明确是否是开始或结束时间，也没有提供...  \n",
       "226  问题描述不够具体，没有提供足够的信息让考生无需查阅图18就能理解问题。答案直接回答了问题，但...  \n",
       "258  问题询问的是未来的行业趋势，这需要考生对金融市场的深入理解和预测，属于较好的开放性问题。但是...  \n",
       "297  问题询问的是一个具体国家的特定事件，涉及实时信息，符合好的问题标准。但答案给出了两个国家，且...  \n",
       "317  问题询问的是具体事件，属于实时信息的范畴，是好的问题。但答案没有提供实际信息，只是说明了文中...  \n",
       "318  问题询问的是具体排名，但答案没有提供明确的顺序或位置，只是模糊地说明位居前列。好的答案应该给...  "
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa_df[qa_df['score'] == 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47bec872-26ec-4265-830e-2a7024f78e6a",
   "metadata": {
    "id": "47bec872-26ec-4265-830e-2a7024f78e6a",
    "outputId": "c1221b16-77e3-40d4-9b27-89dbf9f230a8"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>context</th>\n",
       "      <th>doc</th>\n",
       "      <th>qa_type</th>\n",
       "      <th>score</th>\n",
       "      <th>score_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>2800873d-c399-4c23-9721-9238e2c941f3</td>\n",
       "      <td>消费者借贷能力和意愿受到什么因素的影响？</td>\n",
       "      <td>美国家庭债务余额拖欠率回升至3%</td>\n",
       "      <td>美国家庭债务余额拖欠率回升至3%，消费者借贷能力和意愿将有所下降。</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 11 2024年\\n响2024年美国经济走势的重要因...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>2</td>\n",
       "      <td>问题询问的是影响消费者借贷能力和意愿的因素，而答案提供的是一条关于美国家庭债务余额拖欠率的具...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    uuid              question  \\\n",
       "91  2800873d-c399-4c23-9721-9238e2c941f3  消费者借贷能力和意愿受到什么因素的影响？   \n",
       "\n",
       "              answer                            context  \\\n",
       "91  美国家庭债务余额拖欠率回升至3%  美国家庭债务余额拖欠率回升至3%，消费者借贷能力和意愿将有所下降。   \n",
       "\n",
       "                                                  doc   qa_type  score  \\\n",
       "91  全球经济金融展望报告\\n中国银行研究院 11 2024年\\n响2024年美国经济走势的重要因...  detailed      2   \n",
       "\n",
       "                                         score_reason  \n",
       "91  问题询问的是影响消费者借贷能力和意愿的因素，而答案提供的是一条关于美国家庭债务余额拖欠率的具...  "
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa_df[qa_df['score'] == 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75d5e694-823e-43c1-90ee-b085d8d818e2",
   "metadata": {
    "id": "75d5e694-823e-43c1-90ee-b085d8d818e2",
    "outputId": "234893ed-148c-4bcc-df7e-2c324103fd6f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>context</th>\n",
       "      <th>doc</th>\n",
       "      <th>qa_type</th>\n",
       "      <th>score</th>\n",
       "      <th>score_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [uuid, question, answer, context, doc, qa_type, score, score_reason]\n",
       "Index: []"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa_df[qa_df['score'] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7006df8c-b6dc-4133-8211-00bc4d44f208",
   "metadata": {
    "id": "7006df8c-b6dc-4133-8211-00bc4d44f208"
   },
   "outputs": [],
   "source": [
    "hq_qa_df = qa_df[qa_df['score'] >= 4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2acc6df0-c856-4a70-b59e-2258d980f3e0",
   "metadata": {
    "id": "2acc6df0-c856-4a70-b59e-2258d980f3e0",
    "outputId": "c7efdcfc-d92e-466f-f431-38536850104c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(390, 8)"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hq_qa_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c14fa29-262f-449c-9c76-2b1240df7851",
   "metadata": {
    "id": "9c14fa29-262f-449c-9c76-2b1240df7851"
   },
   "outputs": [],
   "source": [
    "test_q = hq_qa_df.sample(100, replace=False)['question'].values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07d3b615-0078-415f-9ec3-c2ea3c1a439a",
   "metadata": {
    "id": "07d3b615-0078-415f-9ec3-c2ea3c1a439a",
    "outputId": "eed511b5-7124-4e5b-b54d-912e6cd94400"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3123169/2938610661.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  hq_qa_df['dataset'] = 'train'\n"
     ]
    }
   ],
   "source": [
    "hq_qa_df['dataset'] = 'train'\n",
    "hq_qa_df.loc[hq_qa_df['question'].isin(test_q), 'dataset'] = 'test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f9802d2-6926-42d9-8acf-a6c725a6ce59",
   "metadata": {
    "id": "1f9802d2-6926-42d9-8acf-a6c725a6ce59",
    "outputId": "ee57d017-b876-488d-c346-09141e901d0e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>context</th>\n",
       "      <th>doc</th>\n",
       "      <th>qa_type</th>\n",
       "      <th>score</th>\n",
       "      <th>score_reason</th>\n",
       "      <th>dataset</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>883b0471-5082-44f4-b625-86386102bc2d</td>\n",
       "      <td>在当前背景下，哪些国家需要发展可再生能源？</td>\n",
       "      <td>沙特阿拉伯、阿联酋、科威特</td>\n",
       "      <td>沙特阿拉伯、阿联酋、科威特等国依赖油气出口的经济增长模式难以为继，上述国家逐渐加快推动可再生...</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 20 2024年\\n图13：部分亚洲新兴经济体GDP...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>4</td>\n",
       "      <td>问题询问的是在当前背景下需要发展可再生能源的国家，答案直接给出了三个具体的国家名称，直接回答...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303</th>\n",
       "      <td>c6fc6079-0daf-4d6a-bd6e-7d9820012425</td>\n",
       "      <td>阿联酋计划到哪一年将吸引外资规模提升至1500亿美元？</td>\n",
       "      <td>2031年</td>\n",
       "      <td>以期到2031年将吸引外资规模提升至1500亿美元。</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 42 2024年\\n供了空间，进一步坚定了海湾六国经...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>5</td>\n",
       "      <td>问题询问的是具体的时间信息，是一个有明确答案的问题；答案直接给出了具体年份，符合要求，没有提...</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>c6fc6079-0daf-4d6a-bd6e-7d9820012425</td>\n",
       "      <td>阿曼的绿氢战略预计需要多少投资额？</td>\n",
       "      <td>1400亿美元</td>\n",
       "      <td>阿曼推出的绿氢战略计划到2050年投资1400亿美元。</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 42 2024年\\n供了空间，进一步坚定了海湾六国经...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>5</td>\n",
       "      <td>问题询问具体的数据，是开放性问题；答案直接给出了数值，无需查找原文，符合好答案的要求。</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>2800873d-c399-4c23-9721-9238e2c941f3</td>\n",
       "      <td>高利率对美国家庭债务负担有什么影响？</td>\n",
       "      <td>增加了美国家庭债务负担</td>\n",
       "      <td>2023年利率上升增加了美国家庭债务负担，家庭债务支出/可支配收入虽低于历史水平，但呈现上升态势。</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 11 2024年\\n响2024年美国经济走势的重要因...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>5</td>\n",
       "      <td>问题询问的是利率变化对美国家庭债务的实际影响，这是一个开放式的问题，需要理解并分析经济原理。...</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>8cf24c61-8169-41e5-8e05-870bf85a4bc8</td>\n",
       "      <td>中国的“一带一路”合作文件覆盖了多少国家和人口的比例？</td>\n",
       "      <td>覆盖了全球超过七成的国家和六成的人口。</td>\n",
       "      <td>152个国家和32个国际组织已与中国签署“一带一路”合作文件，覆盖全球超过七成的国家和六成的人口。</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 21 2024年\\n专栏2：全球发展中国家合作进展\\...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>5</td>\n",
       "      <td>问题询问的是具体的数据比例，答案直接给出了准确的比例，没有引用原文，符合好的问题和答案的标准。</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>8cf24c61-8169-41e5-8e05-870bf85a4bc8</td>\n",
       "      <td>中国在绿色发展方面的举措是什么？</td>\n",
       "      <td>中国将“促进绿色发展”列为高质量共建“一带一路”行动之一，并计划在非洲和东南亚成立区域办公室...</td>\n",
       "      <td>中国将“促进绿色发展”列为高质量共建“一带一路”八项行动之一，将在非洲和东南亚成立区域办公室...</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 21 2024年\\n专栏2：全球发展中国家合作进展\\...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>4</td>\n",
       "      <td>问题询问的是具体的举措，答案直接提供了中国的绿色发展措施，内容具体且直接回答了问题。但答案稍...</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>883b0471-5082-44f4-b625-86386102bc2d</td>\n",
       "      <td>地缘政治冲突对全球经济运行产生了什么影响？</td>\n",
       "      <td>全球经济运行的不确定性增加</td>\n",
       "      <td>当前，俄乌、巴以等地缘政治冲突持续，全球经济运行的不确定性增加。</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 20 2024年\\n图13：部分亚洲新兴经济体GDP...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>4</td>\n",
       "      <td>问题询问的是具体的影响，答案直接给出了因地缘政治冲突带来的经济影响，即增加了全球经济运行的不...</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>哪些热点问题在报告中被提及？</td>\n",
       "      <td>海湾六国经济发展与投资前景，高利率和高债务对美国房地产市场脆弱性的影响。</td>\n",
       "      <td>海湾六国经济发展与投资前景、高利率和高债务对美国房地产市场脆弱性的影响等热点问题值得关注。</td>\n",
       "      <td>研究院\\n全球经济金融展望报告\\n要点2024年年报（总第57期） 报告日期：2023年12...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>4</td>\n",
       "      <td>问题询问的是具体被提及的热点问题，答案直接给出了这些问题的名称，没有直接引用原文，回答具有针...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>d58699ac-79da-4321-98c5-56c0350d2ba5</td>\n",
       "      <td>美欧加大国债发行力度可能导致什么后果？</td>\n",
       "      <td>增加财政体系的脆弱性</td>\n",
       "      <td>未来随着美欧加大国债发行力度，将进一步增加财政体系的脆弱性。</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 35 2024年\\n体国债收益率出现较大幅度下跌。当...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>5</td>\n",
       "      <td>问题询问的是国债发行可能带来的后果，这需要考生理解和分析经济政策的影响。答案直接给出了一个合...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>3c0da094-8444-428b-9729-f1caa33e4b06</td>\n",
       "      <td>2023年前两个季度巴西的GDP增速超过了多少？</td>\n",
       "      <td>3%</td>\n",
       "      <td>2023年前两个季度巴西和墨西哥GDP增速均超过3%，阿根廷则面临高通胀与负增长并存的困境。</td>\n",
       "      <td>全球经济金融展望报告\\n中国银行研究院 19 2024年\\n新兴经济体增长的主要因素，三季度...</td>\n",
       "      <td>detailed</td>\n",
       "      <td>5</td>\n",
       "      <td>问题询问具体的数据，是可验证的事实；答案直接给出了数值，符合要求。</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     uuid                     question  \\\n",
       "154  883b0471-5082-44f4-b625-86386102bc2d        在当前背景下，哪些国家需要发展可再生能源？   \n",
       "303  c6fc6079-0daf-4d6a-bd6e-7d9820012425  阿联酋计划到哪一年将吸引外资规模提升至1500亿美元？   \n",
       "300  c6fc6079-0daf-4d6a-bd6e-7d9820012425            阿曼的绿氢战略预计需要多少投资额？   \n",
       "90   2800873d-c399-4c23-9721-9238e2c941f3           高利率对美国家庭债务负担有什么影响？   \n",
       "167  8cf24c61-8169-41e5-8e05-870bf85a4bc8  中国的“一带一路”合作文件覆盖了多少国家和人口的比例？   \n",
       "164  8cf24c61-8169-41e5-8e05-870bf85a4bc8             中国在绿色发展方面的举措是什么？   \n",
       "159  883b0471-5082-44f4-b625-86386102bc2d        地缘政治冲突对全球经济运行产生了什么影响？   \n",
       "6    e73a0c9d-d42b-4350-a4c3-b38bf67c68a5               哪些热点问题在报告中被提及？   \n",
       "262  d58699ac-79da-4321-98c5-56c0350d2ba5          美欧加大国债发行力度可能导致什么后果？   \n",
       "146  3c0da094-8444-428b-9729-f1caa33e4b06     2023年前两个季度巴西的GDP增速超过了多少？   \n",
       "\n",
       "                                                answer  \\\n",
       "154                                      沙特阿拉伯、阿联酋、科威特   \n",
       "303                                              2031年   \n",
       "300                                            1400亿美元   \n",
       "90                                         增加了美国家庭债务负担   \n",
       "167                                覆盖了全球超过七成的国家和六成的人口。   \n",
       "164  中国将“促进绿色发展”列为高质量共建“一带一路”行动之一，并计划在非洲和东南亚成立区域办公室...   \n",
       "159                                      全球经济运行的不确定性增加   \n",
       "6                 海湾六国经济发展与投资前景，高利率和高债务对美国房地产市场脆弱性的影响。   \n",
       "262                                         增加财政体系的脆弱性   \n",
       "146                                                 3%   \n",
       "\n",
       "                                               context  \\\n",
       "154  沙特阿拉伯、阿联酋、科威特等国依赖油气出口的经济增长模式难以为继，上述国家逐渐加快推动可再生...   \n",
       "303                         以期到2031年将吸引外资规模提升至1500亿美元。   \n",
       "300                        阿曼推出的绿氢战略计划到2050年投资1400亿美元。   \n",
       "90   2023年利率上升增加了美国家庭债务负担，家庭债务支出/可支配收入虽低于历史水平，但呈现上升态势。   \n",
       "167  152个国家和32个国际组织已与中国签署“一带一路”合作文件，覆盖全球超过七成的国家和六成的人口。   \n",
       "164  中国将“促进绿色发展”列为高质量共建“一带一路”八项行动之一，将在非洲和东南亚成立区域办公室...   \n",
       "159                   当前，俄乌、巴以等地缘政治冲突持续，全球经济运行的不确定性增加。   \n",
       "6        海湾六国经济发展与投资前景、高利率和高债务对美国房地产市场脆弱性的影响等热点问题值得关注。   \n",
       "262                     未来随着美欧加大国债发行力度，将进一步增加财政体系的脆弱性。   \n",
       "146     2023年前两个季度巴西和墨西哥GDP增速均超过3%，阿根廷则面临高通胀与负增长并存的困境。   \n",
       "\n",
       "                                                   doc   qa_type  score  \\\n",
       "154  全球经济金融展望报告\\n中国银行研究院 20 2024年\\n图13：部分亚洲新兴经济体GDP...  detailed      4   \n",
       "303  全球经济金融展望报告\\n中国银行研究院 42 2024年\\n供了空间，进一步坚定了海湾六国经...  detailed      5   \n",
       "300  全球经济金融展望报告\\n中国银行研究院 42 2024年\\n供了空间，进一步坚定了海湾六国经...  detailed      5   \n",
       "90   全球经济金融展望报告\\n中国银行研究院 11 2024年\\n响2024年美国经济走势的重要因...  detailed      5   \n",
       "167  全球经济金融展望报告\\n中国银行研究院 21 2024年\\n专栏2：全球发展中国家合作进展\\...  detailed      5   \n",
       "164  全球经济金融展望报告\\n中国银行研究院 21 2024年\\n专栏2：全球发展中国家合作进展\\...  detailed      4   \n",
       "159  全球经济金融展望报告\\n中国银行研究院 20 2024年\\n图13：部分亚洲新兴经济体GDP...  detailed      4   \n",
       "6    研究院\\n全球经济金融展望报告\\n要点2024年年报（总第57期） 报告日期：2023年12...  detailed      4   \n",
       "262  全球经济金融展望报告\\n中国银行研究院 35 2024年\\n体国债收益率出现较大幅度下跌。当...  detailed      5   \n",
       "146  全球经济金融展望报告\\n中国银行研究院 19 2024年\\n新兴经济体增长的主要因素，三季度...  detailed      5   \n",
       "\n",
       "                                          score_reason dataset  \n",
       "154  问题询问的是在当前背景下需要发展可再生能源的国家，答案直接给出了三个具体的国家名称，直接回答...    test  \n",
       "303  问题询问的是具体的时间信息，是一个有明确答案的问题；答案直接给出了具体年份，符合要求，没有提...   train  \n",
       "300        问题询问具体的数据，是开放性问题；答案直接给出了数值，无需查找原文，符合好答案的要求。   train  \n",
       "90   问题询问的是利率变化对美国家庭债务的实际影响，这是一个开放式的问题，需要理解并分析经济原理。...   train  \n",
       "167    问题询问的是具体的数据比例，答案直接给出了准确的比例，没有引用原文，符合好的问题和答案的标准。   train  \n",
       "164  问题询问的是具体的举措，答案直接提供了中国的绿色发展措施，内容具体且直接回答了问题。但答案稍...   train  \n",
       "159  问题询问的是具体的影响，答案直接给出了因地缘政治冲突带来的经济影响，即增加了全球经济运行的不...   train  \n",
       "6    问题询问的是具体被提及的热点问题，答案直接给出了这些问题的名称，没有直接引用原文，回答具有针...    test  \n",
       "262  问题询问的是国债发行可能带来的后果，这需要考生理解和分析经济政策的影响。答案直接给出了一个合...    test  \n",
       "146                  问题询问具体的数据，是可验证的事实；答案直接给出了数值，符合要求。   train  "
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hq_qa_df.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "952b1457-1b1b-49dc-8df3-ceb505faa1b5",
   "metadata": {
    "id": "952b1457-1b1b-49dc-8df3-ceb505faa1b5"
   },
   "outputs": [],
   "source": [
    "hq_qa_df.to_excel(os.path.join(output_dir, f'question_answer.xlsx'), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "217082f7-bcd6-418f-afd5-d36ca3485691",
   "metadata": {
    "id": "217082f7-bcd6-418f-afd5-d36ca3485691"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "97721fe8e229406daa546d30f6b1f3c4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3a1483b980ff40a4b113c9adb1debde7",
       "IPY_MODEL_14f5510c4b9a41f7974b8ef701fe2ac2",
       "IPY_MODEL_070d1468c7264d2d98172e1d75acd2f4"
      ],
      "layout": "IPY_MODEL_f64c0f996f474cd4ab55b9faa4973f2d"
     }
    },
    "3a1483b980ff40a4b113c9adb1debde7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_07bd5384c9314c46aa63d2950d655f9a",
      "placeholder": "​",
      "style": "IPY_MODEL_7726409007ab4afdb345b00002dc0b00",
      "value": "100%"
     }
    },
    "14f5510c4b9a41f7974b8ef701fe2ac2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b9fb90215c684f2d8cdbc80ac81ba9e2",
      "max": 49,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_a4ccc283659d4794a6ef436a9ec372e1",
      "value": 49
     }
    },
    "070d1468c7264d2d98172e1d75acd2f4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a5b98dd061c9476b897e7f6df03d285d",
      "placeholder": "​",
      "style": "IPY_MODEL_2c24223263dd4df1b5305dc878358fff",
      "value": " 49/49 [05:07&lt;00:00,  7.98s/it]"
     }
    },
    "f64c0f996f474cd4ab55b9faa4973f2d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "07bd5384c9314c46aa63d2950d655f9a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7726409007ab4afdb345b00002dc0b00": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b9fb90215c684f2d8cdbc80ac81ba9e2": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a4ccc283659d4794a6ef436a9ec372e1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "a5b98dd061c9476b897e7f6df03d285d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2c24223263dd4df1b5305dc878358fff": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "513768f81b094f358168994ad03f9cc2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_219d202b5f31479988b4dd52010bdd6e",
       "IPY_MODEL_72376297dce54eb2a47e4526e6906f88",
       "IPY_MODEL_2cd71a6e921f4073bc415fb31f7b63fc"
      ],
      "layout": "IPY_MODEL_b0c2870b0f014c4793a232bbbb2f1e03"
     }
    },
    "219d202b5f31479988b4dd52010bdd6e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e02a2b231add4980a5d6e52244902ff0",
      "placeholder": "​",
      "style": "IPY_MODEL_5c222bdd56054b69b708eda14f4cce51",
      "value": "100%"
     }
    },
    "72376297dce54eb2a47e4526e6906f88": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e67f4a688ebe490e9d0e866252136885",
      "max": 28,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_d887ab0146b94f399d413b68bc404e3d",
      "value": 28
     }
    },
    "2cd71a6e921f4073bc415fb31f7b63fc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c369641245b14114a3c621f6969ccfcd",
      "placeholder": "​",
      "style": "IPY_MODEL_1d559d3c3b46467b8b6d6cc8193d6b5b",
      "value": " 28/28 [01:04&lt;00:00,  1.96s/it]"
     }
    },
    "b0c2870b0f014c4793a232bbbb2f1e03": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e02a2b231add4980a5d6e52244902ff0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5c222bdd56054b69b708eda14f4cce51": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e67f4a688ebe490e9d0e866252136885": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d887ab0146b94f399d413b68bc404e3d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "c369641245b14114a3c621f6969ccfcd": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1d559d3c3b46467b8b6d6cc8193d6b5b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}