research_questions/emse/RQ5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ab79d354",
   "metadata": {},
   "source": [
    "# What is the effectiveness of the pre-trained model in commit message generation?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fedc7d79",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import itertools, os,json\n",
    "\n",
    "model_name = \"CodeBERT\"\n",
    "dataset_name = \"MCMD\"\n",
    "lan_list = [\"java\", \"python\", \"javascript\", \"cpp\", \"csharp\"]\n",
    "metric_list = ['B-Moses', 'B-Norm', 'B-CC', 'Rouge-1', 'Rouge-2', 'Rouge-L', 'Meteor']\n",
    "display_lan_list = lan_list\n",
    "lan_list.sort()\n",
    "\n",
    "def comb_finetune(test_pl, lan_n):\n",
    "    results = pd.DataFrame(columns=metric_list, index=list(itertools.combinations(lan_list, lan_n)))\n",
    "    for comb in itertools.combinations(lan_list, lan_n):\n",
    "        finetune_pl_comb = \"_\".join(comb)\n",
    "        ref_path = \"../experiment_results/model_CodeBERT/data_MCMD/{}/ref.msg\".format(test_pl)\n",
    "        if lan_n == 1 and test_pl==finetune_pl_comb:\n",
    "            gen_path = \"../experiment_results/model_CodeBERT/data_MCMD/{}/gen.msg\".format(test_pl, finetune_pl_comb)\n",
    "        else:\n",
    "            gen_path = \"../experiment_results/model_CodeBERT/data_MCMD/{}/finetune_{}/gen.msg\".format(test_pl, finetune_pl_comb)\n",
    "        BM_evaluate_cmd = \"../metrics/B-Moses.perl {} < {}\".format(ref_path, gen_path)\n",
    "        results['B-Moses'][comb] = float(os.popen(BM_evaluate_cmd).read().split(\"=\")[1].split(\",\")[0].strip())\n",
    "        BN_evaluate_cmd = \"python ../metrics/B-Norm.py {} < {}\".format(ref_path, gen_path)\n",
    "        results['B-Norm'][comb] = float(os.popen(BN_evaluate_cmd).read())\n",
    "        BC_evaluate_cmd = \"python ../metrics/B-CC.py --ref_path {} --gen_path {}\".format(ref_path, gen_path)\n",
    "        results['B-CC'][comb] = float(os.popen(BC_evaluate_cmd).read().strip())\n",
    "        Rouge_evaluate_cmd = \"python ../metrics/Rouge.py --ref_path {} --gen_path {}\".format(ref_path, gen_path)\n",
    "        rouge_str = os.popen(Rouge_evaluate_cmd).read().replace(\"'\", \"\\\"\")\n",
    "        rouge_dict = json.loads(rouge_str)\n",
    "        results['Rouge-1'][comb], results['Rouge-2'][comb], results['Rouge-L'][comb] = rouge_dict['ROUGE-1'], rouge_dict['ROUGE-2'], rouge_dict['ROUGE-L']\n",
    "        Meteor_evaluate_cmd = \"python ../metrics/Meteor.py --ref_path {} --gen_path {}\".format(ref_path, gen_path)\n",
    "        results['Meteor'][comb] = float(os.popen(Meteor_evaluate_cmd).read().strip())\n",
    "    pd.options.display.float_format = '{:.2f}'.format\n",
    "    return results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f1f3ba4",
   "metadata": {},
   "source": [
    "## Test PL: Java"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccfe9a72",
   "metadata": {},
   "source": [
    "### MCMD<sub>1-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "db68d464",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp,)</th>\n",
       "      <td>2.04</td>\n",
       "      <td>7.08</td>\n",
       "      <td>3.26</td>\n",
       "      <td>9.23</td>\n",
       "      <td>3.07</td>\n",
       "      <td>9.01</td>\n",
       "      <td>6.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp,)</th>\n",
       "      <td>2.21</td>\n",
       "      <td>6.82</td>\n",
       "      <td>3.61</td>\n",
       "      <td>8.83</td>\n",
       "      <td>3.08</td>\n",
       "      <td>8.65</td>\n",
       "      <td>7.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java,)</th>\n",
       "      <td>10.79</td>\n",
       "      <td>18.98</td>\n",
       "      <td>13.02</td>\n",
       "      <td>22.74</td>\n",
       "      <td>11.95</td>\n",
       "      <td>22.34</td>\n",
       "      <td>20.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(javascript,)</th>\n",
       "      <td>1.93</td>\n",
       "      <td>6.92</td>\n",
       "      <td>3.14</td>\n",
       "      <td>9.72</td>\n",
       "      <td>2.82</td>\n",
       "      <td>9.43</td>\n",
       "      <td>7.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(python,)</th>\n",
       "      <td>2.32</td>\n",
       "      <td>7.06</td>\n",
       "      <td>3.39</td>\n",
       "      <td>10.31</td>\n",
       "      <td>3.49</td>\n",
       "      <td>10.02</td>\n",
       "      <td>7.79</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L Meteor\n",
       "(cpp,)           2.04   7.08  3.26    9.23    3.07    9.01   6.98\n",
       "(csharp,)        2.21   6.82  3.61    8.83    3.08    8.65   7.70\n",
       "(java,)         10.79  18.98 13.02   22.74   11.95   22.34  20.99\n",
       "(javascript,)    1.93   6.92  3.14    9.72    2.82    9.43   7.47\n",
       "(python,)        2.32   7.06  3.39   10.31    3.49   10.02   7.79"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"java\",1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4e3ca45",
   "metadata": {},
   "source": [
    "### MCMD<sub>2-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7ce4aec8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp)</th>\n",
       "      <td>3.08</td>\n",
       "      <td>7.64</td>\n",
       "      <td>4.05</td>\n",
       "      <td>10.13</td>\n",
       "      <td>3.85</td>\n",
       "      <td>9.87</td>\n",
       "      <td>8.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java)</th>\n",
       "      <td>10.54</td>\n",
       "      <td>18.56</td>\n",
       "      <td>12.76</td>\n",
       "      <td>21.97</td>\n",
       "      <td>11.70</td>\n",
       "      <td>21.60</td>\n",
       "      <td>20.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, javascript)</th>\n",
       "      <td>2.41</td>\n",
       "      <td>7.05</td>\n",
       "      <td>3.52</td>\n",
       "      <td>8.71</td>\n",
       "      <td>3.16</td>\n",
       "      <td>8.45</td>\n",
       "      <td>6.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, python)</th>\n",
       "      <td>2.50</td>\n",
       "      <td>7.39</td>\n",
       "      <td>3.70</td>\n",
       "      <td>9.59</td>\n",
       "      <td>3.65</td>\n",
       "      <td>9.30</td>\n",
       "      <td>7.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java)</th>\n",
       "      <td>10.78</td>\n",
       "      <td>18.76</td>\n",
       "      <td>12.97</td>\n",
       "      <td>22.27</td>\n",
       "      <td>11.84</td>\n",
       "      <td>21.89</td>\n",
       "      <td>20.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, javascript)</th>\n",
       "      <td>2.42</td>\n",
       "      <td>7.28</td>\n",
       "      <td>3.45</td>\n",
       "      <td>10.33</td>\n",
       "      <td>3.37</td>\n",
       "      <td>10.01</td>\n",
       "      <td>8.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, python)</th>\n",
       "      <td>2.46</td>\n",
       "      <td>7.66</td>\n",
       "      <td>3.78</td>\n",
       "      <td>10.50</td>\n",
       "      <td>3.42</td>\n",
       "      <td>10.18</td>\n",
       "      <td>8.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, javascript)</th>\n",
       "      <td>10.75</td>\n",
       "      <td>18.86</td>\n",
       "      <td>13.00</td>\n",
       "      <td>22.41</td>\n",
       "      <td>11.93</td>\n",
       "      <td>22.01</td>\n",
       "      <td>20.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, python)</th>\n",
       "      <td>10.81</td>\n",
       "      <td>18.89</td>\n",
       "      <td>13.04</td>\n",
       "      <td>22.47</td>\n",
       "      <td>11.94</td>\n",
       "      <td>22.06</td>\n",
       "      <td>20.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(javascript, python)</th>\n",
       "      <td>2.46</td>\n",
       "      <td>7.90</td>\n",
       "      <td>3.75</td>\n",
       "      <td>11.40</td>\n",
       "      <td>3.64</td>\n",
       "      <td>11.00</td>\n",
       "      <td>8.79</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L Meteor\n",
       "(cpp, csharp)           3.08   7.64  4.05   10.13    3.85    9.87   8.10\n",
       "(cpp, java)            10.54  18.56 12.76   21.97   11.70   21.60  20.35\n",
       "(cpp, javascript)       2.41   7.05  3.52    8.71    3.16    8.45   6.94\n",
       "(cpp, python)           2.50   7.39  3.70    9.59    3.65    9.30   7.41\n",
       "(csharp, java)         10.78  18.76 12.97   22.27   11.84   21.89  20.71\n",
       "(csharp, javascript)    2.42   7.28  3.45   10.33    3.37   10.01   8.15\n",
       "(csharp, python)        2.46   7.66  3.78   10.50    3.42   10.18   8.53\n",
       "(java, javascript)     10.75  18.86 13.00   22.41   11.93   22.01  20.82\n",
       "(java, python)         10.81  18.89 13.04   22.47   11.94   22.06  20.88\n",
       "(javascript, python)    2.46   7.90  3.75   11.40    3.64   11.00   8.79"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"java\",2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b01d9c1",
   "metadata": {},
   "source": [
    "### MCMD<sub>3-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "40f6dabe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java)</th>\n",
       "      <td>10.76</td>\n",
       "      <td>18.58</td>\n",
       "      <td>12.87</td>\n",
       "      <td>22.09</td>\n",
       "      <td>11.72</td>\n",
       "      <td>21.70</td>\n",
       "      <td>20.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, javascript)</th>\n",
       "      <td>2.66</td>\n",
       "      <td>7.44</td>\n",
       "      <td>3.76</td>\n",
       "      <td>9.41</td>\n",
       "      <td>3.58</td>\n",
       "      <td>9.12</td>\n",
       "      <td>7.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, python)</th>\n",
       "      <td>2.89</td>\n",
       "      <td>7.83</td>\n",
       "      <td>4.10</td>\n",
       "      <td>10.09</td>\n",
       "      <td>3.84</td>\n",
       "      <td>9.77</td>\n",
       "      <td>7.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, javascript)</th>\n",
       "      <td>10.94</td>\n",
       "      <td>19.00</td>\n",
       "      <td>13.18</td>\n",
       "      <td>22.72</td>\n",
       "      <td>12.11</td>\n",
       "      <td>22.29</td>\n",
       "      <td>21.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, python)</th>\n",
       "      <td>10.81</td>\n",
       "      <td>18.86</td>\n",
       "      <td>13.05</td>\n",
       "      <td>22.53</td>\n",
       "      <td>12.05</td>\n",
       "      <td>22.12</td>\n",
       "      <td>20.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, javascript, python)</th>\n",
       "      <td>2.55</td>\n",
       "      <td>7.40</td>\n",
       "      <td>3.75</td>\n",
       "      <td>8.95</td>\n",
       "      <td>3.35</td>\n",
       "      <td>8.66</td>\n",
       "      <td>7.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, javascript)</th>\n",
       "      <td>10.92</td>\n",
       "      <td>18.91</td>\n",
       "      <td>13.11</td>\n",
       "      <td>22.57</td>\n",
       "      <td>12.05</td>\n",
       "      <td>22.16</td>\n",
       "      <td>20.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, python)</th>\n",
       "      <td>10.70</td>\n",
       "      <td>18.67</td>\n",
       "      <td>12.84</td>\n",
       "      <td>22.25</td>\n",
       "      <td>11.83</td>\n",
       "      <td>21.87</td>\n",
       "      <td>20.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, javascript, python)</th>\n",
       "      <td>2.64</td>\n",
       "      <td>8.33</td>\n",
       "      <td>4.04</td>\n",
       "      <td>11.41</td>\n",
       "      <td>3.69</td>\n",
       "      <td>11.02</td>\n",
       "      <td>9.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, javascript, python)</th>\n",
       "      <td>10.83</td>\n",
       "      <td>18.77</td>\n",
       "      <td>13.02</td>\n",
       "      <td>22.47</td>\n",
       "      <td>11.98</td>\n",
       "      <td>22.05</td>\n",
       "      <td>20.82</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L  \\\n",
       "(cpp, csharp, java)            10.76  18.58 12.87   22.09   11.72   21.70   \n",
       "(cpp, csharp, javascript)       2.66   7.44  3.76    9.41    3.58    9.12   \n",
       "(cpp, csharp, python)           2.89   7.83  4.10   10.09    3.84    9.77   \n",
       "(cpp, java, javascript)        10.94  19.00 13.18   22.72   12.11   22.29   \n",
       "(cpp, java, python)            10.81  18.86 13.05   22.53   12.05   22.12   \n",
       "(cpp, javascript, python)       2.55   7.40  3.75    8.95    3.35    8.66   \n",
       "(csharp, java, javascript)     10.92  18.91 13.11   22.57   12.05   22.16   \n",
       "(csharp, java, python)         10.70  18.67 12.84   22.25   11.83   21.87   \n",
       "(csharp, javascript, python)    2.64   8.33  4.04   11.41    3.69   11.02   \n",
       "(java, javascript, python)     10.83  18.77 13.02   22.47   11.98   22.05   \n",
       "\n",
       "                             Meteor  \n",
       "(cpp, csharp, java)           20.52  \n",
       "(cpp, csharp, javascript)      7.35  \n",
       "(cpp, csharp, python)          7.92  \n",
       "(cpp, java, javascript)       21.06  \n",
       "(cpp, java, python)           20.83  \n",
       "(cpp, javascript, python)      7.20  \n",
       "(csharp, java, javascript)    20.91  \n",
       "(csharp, java, python)        20.54  \n",
       "(csharp, javascript, python)   9.32  \n",
       "(java, javascript, python)    20.82  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"java\",3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48167ff9",
   "metadata": {},
   "source": [
    "### MCMD<sub>4-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bd46a849",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, javascript)</th>\n",
       "      <td>10.88</td>\n",
       "      <td>18.58</td>\n",
       "      <td>12.97</td>\n",
       "      <td>22.12</td>\n",
       "      <td>11.85</td>\n",
       "      <td>21.71</td>\n",
       "      <td>20.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, python)</th>\n",
       "      <td>8.57</td>\n",
       "      <td>15.80</td>\n",
       "      <td>10.54</td>\n",
       "      <td>19.23</td>\n",
       "      <td>10.21</td>\n",
       "      <td>18.83</td>\n",
       "      <td>16.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, javascript, python)</th>\n",
       "      <td>2.98</td>\n",
       "      <td>8.29</td>\n",
       "      <td>4.17</td>\n",
       "      <td>11.29</td>\n",
       "      <td>3.94</td>\n",
       "      <td>10.91</td>\n",
       "      <td>9.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, javascript, python)</th>\n",
       "      <td>10.81</td>\n",
       "      <td>18.88</td>\n",
       "      <td>12.99</td>\n",
       "      <td>22.60</td>\n",
       "      <td>11.89</td>\n",
       "      <td>22.15</td>\n",
       "      <td>20.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, javascript, python)</th>\n",
       "      <td>10.90</td>\n",
       "      <td>18.93</td>\n",
       "      <td>13.07</td>\n",
       "      <td>22.72</td>\n",
       "      <td>12.09</td>\n",
       "      <td>22.28</td>\n",
       "      <td>20.93</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   B-Moses B-Norm  B-CC Rouge-1 Rouge-2  \\\n",
       "(cpp, csharp, java, javascript)      10.88  18.58 12.97   22.12   11.85   \n",
       "(cpp, csharp, java, python)           8.57  15.80 10.54   19.23   10.21   \n",
       "(cpp, csharp, javascript, python)     2.98   8.29  4.17   11.29    3.94   \n",
       "(cpp, java, javascript, python)      10.81  18.88 12.99   22.60   11.89   \n",
       "(csharp, java, javascript, python)   10.90  18.93 13.07   22.72   12.09   \n",
       "\n",
       "                                   Rouge-L Meteor  \n",
       "(cpp, csharp, java, javascript)      21.71  20.62  \n",
       "(cpp, csharp, java, python)          18.83  16.58  \n",
       "(cpp, csharp, javascript, python)    10.91   9.01  \n",
       "(cpp, java, javascript, python)      22.15  20.86  \n",
       "(csharp, java, javascript, python)   22.28  20.93  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"java\",4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30a141dc",
   "metadata": {},
   "source": [
    "### MCMD<sub>5-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cd071782",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, javascript, python)</th>\n",
       "      <td>11.03</td>\n",
       "      <td>19.10</td>\n",
       "      <td>13.24</td>\n",
       "      <td>22.89</td>\n",
       "      <td>12.16</td>\n",
       "      <td>22.43</td>\n",
       "      <td>21.21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        B-Moses B-Norm  B-CC Rouge-1 Rouge-2  \\\n",
       "(cpp, csharp, java, javascript, python)   11.03  19.10 13.24   22.89   12.16   \n",
       "\n",
       "                                        Rouge-L Meteor  \n",
       "(cpp, csharp, java, javascript, python)   22.43  21.21  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"java\",5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9e0e7b0",
   "metadata": {},
   "source": [
    "## Test PL: Python (Py)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35795621",
   "metadata": {},
   "source": [
    "### MCMD<sub>1-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "be98129c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp,)</th>\n",
       "      <td>2.75</td>\n",
       "      <td>9.45</td>\n",
       "      <td>4.09</td>\n",
       "      <td>13.59</td>\n",
       "      <td>5.30</td>\n",
       "      <td>13.23</td>\n",
       "      <td>9.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp,)</th>\n",
       "      <td>1.73</td>\n",
       "      <td>7.65</td>\n",
       "      <td>3.04</td>\n",
       "      <td>10.58</td>\n",
       "      <td>3.70</td>\n",
       "      <td>10.34</td>\n",
       "      <td>7.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java,)</th>\n",
       "      <td>2.10</td>\n",
       "      <td>7.74</td>\n",
       "      <td>3.55</td>\n",
       "      <td>10.70</td>\n",
       "      <td>3.48</td>\n",
       "      <td>10.38</td>\n",
       "      <td>7.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(javascript,)</th>\n",
       "      <td>2.02</td>\n",
       "      <td>8.12</td>\n",
       "      <td>3.30</td>\n",
       "      <td>11.89</td>\n",
       "      <td>3.98</td>\n",
       "      <td>11.57</td>\n",
       "      <td>7.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(python,)</th>\n",
       "      <td>10.55</td>\n",
       "      <td>19.39</td>\n",
       "      <td>12.97</td>\n",
       "      <td>25.28</td>\n",
       "      <td>13.48</td>\n",
       "      <td>24.63</td>\n",
       "      <td>22.69</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L Meteor\n",
       "(cpp,)           2.75   9.45  4.09   13.59    5.30   13.23   9.03\n",
       "(csharp,)        1.73   7.65  3.04   10.58    3.70   10.34   7.38\n",
       "(java,)          2.10   7.74  3.55   10.70    3.48   10.38   7.96\n",
       "(javascript,)    2.02   8.12  3.30   11.89    3.98   11.57   7.91\n",
       "(python,)       10.55  19.39 12.97   25.28   13.48   24.63  22.69"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"python\",1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "498c3ec0",
   "metadata": {},
   "source": [
    "### MCMD<sub>2-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "40bfa2a3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp)</th>\n",
       "      <td>2.97</td>\n",
       "      <td>9.63</td>\n",
       "      <td>4.31</td>\n",
       "      <td>13.59</td>\n",
       "      <td>5.45</td>\n",
       "      <td>13.24</td>\n",
       "      <td>9.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java)</th>\n",
       "      <td>2.90</td>\n",
       "      <td>9.20</td>\n",
       "      <td>4.19</td>\n",
       "      <td>13.51</td>\n",
       "      <td>5.52</td>\n",
       "      <td>13.14</td>\n",
       "      <td>9.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, javascript)</th>\n",
       "      <td>2.95</td>\n",
       "      <td>9.29</td>\n",
       "      <td>4.15</td>\n",
       "      <td>13.03</td>\n",
       "      <td>5.25</td>\n",
       "      <td>12.65</td>\n",
       "      <td>8.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, python)</th>\n",
       "      <td>9.56</td>\n",
       "      <td>17.56</td>\n",
       "      <td>11.60</td>\n",
       "      <td>23.30</td>\n",
       "      <td>12.55</td>\n",
       "      <td>22.69</td>\n",
       "      <td>19.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java)</th>\n",
       "      <td>2.27</td>\n",
       "      <td>8.44</td>\n",
       "      <td>3.65</td>\n",
       "      <td>12.25</td>\n",
       "      <td>4.50</td>\n",
       "      <td>11.88</td>\n",
       "      <td>8.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, javascript)</th>\n",
       "      <td>2.44</td>\n",
       "      <td>8.70</td>\n",
       "      <td>3.78</td>\n",
       "      <td>12.90</td>\n",
       "      <td>4.84</td>\n",
       "      <td>12.50</td>\n",
       "      <td>8.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, python)</th>\n",
       "      <td>10.84</td>\n",
       "      <td>19.33</td>\n",
       "      <td>13.11</td>\n",
       "      <td>25.01</td>\n",
       "      <td>13.48</td>\n",
       "      <td>24.35</td>\n",
       "      <td>22.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, javascript)</th>\n",
       "      <td>2.56</td>\n",
       "      <td>9.27</td>\n",
       "      <td>3.94</td>\n",
       "      <td>13.71</td>\n",
       "      <td>4.96</td>\n",
       "      <td>13.27</td>\n",
       "      <td>9.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, python)</th>\n",
       "      <td>10.66</td>\n",
       "      <td>19.28</td>\n",
       "      <td>12.98</td>\n",
       "      <td>25.02</td>\n",
       "      <td>13.45</td>\n",
       "      <td>24.36</td>\n",
       "      <td>22.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(javascript, python)</th>\n",
       "      <td>10.87</td>\n",
       "      <td>19.47</td>\n",
       "      <td>13.16</td>\n",
       "      <td>25.33</td>\n",
       "      <td>13.53</td>\n",
       "      <td>24.64</td>\n",
       "      <td>22.80</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L Meteor\n",
       "(cpp, csharp)           2.97   9.63  4.31   13.59    5.45   13.24   9.37\n",
       "(cpp, java)             2.90   9.20  4.19   13.51    5.52   13.14   9.18\n",
       "(cpp, javascript)       2.95   9.29  4.15   13.03    5.25   12.65   8.47\n",
       "(cpp, python)           9.56  17.56 11.60   23.30   12.55   22.69  19.59\n",
       "(csharp, java)          2.27   8.44  3.65   12.25    4.50   11.88   8.37\n",
       "(csharp, javascript)    2.44   8.70  3.78   12.90    4.84   12.50   8.63\n",
       "(csharp, python)       10.84  19.33 13.11   25.01   13.48   24.35  22.70\n",
       "(java, javascript)      2.56   9.27  3.94   13.71    4.96   13.27   9.05\n",
       "(java, python)         10.66  19.28 12.98   25.02   13.45   24.36  22.64\n",
       "(javascript, python)   10.87  19.47 13.16   25.33   13.53   24.64  22.80"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"python\",2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee12692b",
   "metadata": {},
   "source": [
    "### MCMD<sub>3-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5b615fd1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java)</th>\n",
       "      <td>2.67</td>\n",
       "      <td>9.02</td>\n",
       "      <td>3.97</td>\n",
       "      <td>13.23</td>\n",
       "      <td>5.08</td>\n",
       "      <td>12.84</td>\n",
       "      <td>8.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, javascript)</th>\n",
       "      <td>3.15</td>\n",
       "      <td>9.81</td>\n",
       "      <td>4.40</td>\n",
       "      <td>13.54</td>\n",
       "      <td>5.46</td>\n",
       "      <td>13.13</td>\n",
       "      <td>8.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, python)</th>\n",
       "      <td>8.53</td>\n",
       "      <td>16.39</td>\n",
       "      <td>10.34</td>\n",
       "      <td>21.84</td>\n",
       "      <td>11.29</td>\n",
       "      <td>21.14</td>\n",
       "      <td>18.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, javascript)</th>\n",
       "      <td>2.95</td>\n",
       "      <td>9.69</td>\n",
       "      <td>4.32</td>\n",
       "      <td>14.37</td>\n",
       "      <td>5.71</td>\n",
       "      <td>13.90</td>\n",
       "      <td>9.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, python)</th>\n",
       "      <td>10.86</td>\n",
       "      <td>19.26</td>\n",
       "      <td>13.14</td>\n",
       "      <td>25.16</td>\n",
       "      <td>13.63</td>\n",
       "      <td>24.48</td>\n",
       "      <td>22.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, javascript, python)</th>\n",
       "      <td>6.62</td>\n",
       "      <td>14.53</td>\n",
       "      <td>8.24</td>\n",
       "      <td>19.51</td>\n",
       "      <td>9.64</td>\n",
       "      <td>18.90</td>\n",
       "      <td>14.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, javascript)</th>\n",
       "      <td>2.53</td>\n",
       "      <td>9.07</td>\n",
       "      <td>3.91</td>\n",
       "      <td>13.64</td>\n",
       "      <td>5.15</td>\n",
       "      <td>13.20</td>\n",
       "      <td>8.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, python)</th>\n",
       "      <td>10.74</td>\n",
       "      <td>19.23</td>\n",
       "      <td>12.98</td>\n",
       "      <td>25.01</td>\n",
       "      <td>13.46</td>\n",
       "      <td>24.36</td>\n",
       "      <td>22.49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, javascript, python)</th>\n",
       "      <td>11.00</td>\n",
       "      <td>19.56</td>\n",
       "      <td>13.26</td>\n",
       "      <td>25.42</td>\n",
       "      <td>13.69</td>\n",
       "      <td>24.70</td>\n",
       "      <td>22.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, javascript, python)</th>\n",
       "      <td>10.84</td>\n",
       "      <td>19.34</td>\n",
       "      <td>13.15</td>\n",
       "      <td>25.20</td>\n",
       "      <td>13.61</td>\n",
       "      <td>24.52</td>\n",
       "      <td>22.71</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L  \\\n",
       "(cpp, csharp, java)             2.67   9.02  3.97   13.23    5.08   12.84   \n",
       "(cpp, csharp, javascript)       3.15   9.81  4.40   13.54    5.46   13.13   \n",
       "(cpp, csharp, python)           8.53  16.39 10.34   21.84   11.29   21.14   \n",
       "(cpp, java, javascript)         2.95   9.69  4.32   14.37    5.71   13.90   \n",
       "(cpp, java, python)            10.86  19.26 13.14   25.16   13.63   24.48   \n",
       "(cpp, javascript, python)       6.62  14.53  8.24   19.51    9.64   18.90   \n",
       "(csharp, java, javascript)      2.53   9.07  3.91   13.64    5.15   13.20   \n",
       "(csharp, java, python)         10.74  19.23 12.98   25.01   13.46   24.36   \n",
       "(csharp, javascript, python)   11.00  19.56 13.26   25.42   13.69   24.70   \n",
       "(java, javascript, python)     10.84  19.34 13.15   25.20   13.61   24.52   \n",
       "\n",
       "                             Meteor  \n",
       "(cpp, csharp, java)            8.90  \n",
       "(cpp, csharp, javascript)      8.81  \n",
       "(cpp, csharp, python)         18.13  \n",
       "(cpp, java, javascript)        9.65  \n",
       "(cpp, java, python)           22.70  \n",
       "(cpp, javascript, python)     14.62  \n",
       "(csharp, java, javascript)     8.90  \n",
       "(csharp, java, python)        22.49  \n",
       "(csharp, javascript, python)  22.92  \n",
       "(java, javascript, python)    22.71  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"python\",3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb346918",
   "metadata": {},
   "source": [
    "### MCMD<sub>4-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "78826ec9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, javascript)</th>\n",
       "      <td>2.89</td>\n",
       "      <td>9.53</td>\n",
       "      <td>4.25</td>\n",
       "      <td>14.02</td>\n",
       "      <td>5.48</td>\n",
       "      <td>13.60</td>\n",
       "      <td>9.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, python)</th>\n",
       "      <td>8.34</td>\n",
       "      <td>16.42</td>\n",
       "      <td>10.22</td>\n",
       "      <td>21.82</td>\n",
       "      <td>11.17</td>\n",
       "      <td>21.09</td>\n",
       "      <td>18.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, javascript, python)</th>\n",
       "      <td>10.90</td>\n",
       "      <td>19.43</td>\n",
       "      <td>13.13</td>\n",
       "      <td>25.33</td>\n",
       "      <td>13.56</td>\n",
       "      <td>24.62</td>\n",
       "      <td>22.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, javascript, python)</th>\n",
       "      <td>10.97</td>\n",
       "      <td>19.38</td>\n",
       "      <td>13.23</td>\n",
       "      <td>25.25</td>\n",
       "      <td>13.67</td>\n",
       "      <td>24.54</td>\n",
       "      <td>22.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, javascript, python)</th>\n",
       "      <td>11.02</td>\n",
       "      <td>19.41</td>\n",
       "      <td>13.28</td>\n",
       "      <td>25.33</td>\n",
       "      <td>13.75</td>\n",
       "      <td>24.65</td>\n",
       "      <td>22.96</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   B-Moses B-Norm  B-CC Rouge-1 Rouge-2  \\\n",
       "(cpp, csharp, java, javascript)       2.89   9.53  4.25   14.02    5.48   \n",
       "(cpp, csharp, java, python)           8.34  16.42 10.22   21.82   11.17   \n",
       "(cpp, csharp, javascript, python)    10.90  19.43 13.13   25.33   13.56   \n",
       "(cpp, java, javascript, python)      10.97  19.38 13.23   25.25   13.67   \n",
       "(csharp, java, javascript, python)   11.02  19.41 13.28   25.33   13.75   \n",
       "\n",
       "                                   Rouge-L Meteor  \n",
       "(cpp, csharp, java, javascript)      13.60   9.47  \n",
       "(cpp, csharp, java, python)          21.09  18.10  \n",
       "(cpp, csharp, javascript, python)    24.62  22.79  \n",
       "(cpp, java, javascript, python)      24.54  22.76  \n",
       "(csharp, java, javascript, python)   24.65  22.96  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"python\",4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d18a9527",
   "metadata": {},
   "source": [
    "### MCMD<sub>5-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a469a68f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, javascript, python)</th>\n",
       "      <td>11.10</td>\n",
       "      <td>19.58</td>\n",
       "      <td>13.35</td>\n",
       "      <td>25.51</td>\n",
       "      <td>13.73</td>\n",
       "      <td>24.79</td>\n",
       "      <td>22.98</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        B-Moses B-Norm  B-CC Rouge-1 Rouge-2  \\\n",
       "(cpp, csharp, java, javascript, python)   11.10  19.58 13.35   25.51   13.73   \n",
       "\n",
       "                                        Rouge-L Meteor  \n",
       "(cpp, csharp, java, javascript, python)   24.79  22.98  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"python\",5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4026379",
   "metadata": {},
   "source": [
    "## Test PL: JavaScript (Js)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a54be965",
   "metadata": {},
   "source": [
    "### MCMD<sub>1-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b1666dba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp,)</th>\n",
       "      <td>5.63</td>\n",
       "      <td>11.22</td>\n",
       "      <td>6.55</td>\n",
       "      <td>15.83</td>\n",
       "      <td>6.71</td>\n",
       "      <td>15.46</td>\n",
       "      <td>12.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp,)</th>\n",
       "      <td>3.41</td>\n",
       "      <td>9.35</td>\n",
       "      <td>4.83</td>\n",
       "      <td>13.26</td>\n",
       "      <td>5.45</td>\n",
       "      <td>13.01</td>\n",
       "      <td>10.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java,)</th>\n",
       "      <td>3.73</td>\n",
       "      <td>8.93</td>\n",
       "      <td>4.86</td>\n",
       "      <td>12.27</td>\n",
       "      <td>4.64</td>\n",
       "      <td>11.97</td>\n",
       "      <td>10.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(javascript,)</th>\n",
       "      <td>17.68</td>\n",
       "      <td>23.01</td>\n",
       "      <td>17.83</td>\n",
       "      <td>28.14</td>\n",
       "      <td>16.89</td>\n",
       "      <td>27.68</td>\n",
       "      <td>26.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(python,)</th>\n",
       "      <td>5.05</td>\n",
       "      <td>11.04</td>\n",
       "      <td>6.41</td>\n",
       "      <td>15.98</td>\n",
       "      <td>6.94</td>\n",
       "      <td>15.60</td>\n",
       "      <td>12.31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L Meteor\n",
       "(cpp,)           5.63  11.22  6.55   15.83    6.71   15.46  12.59\n",
       "(csharp,)        3.41   9.35  4.83   13.26    5.45   13.01  10.08\n",
       "(java,)          3.73   8.93  4.86   12.27    4.64   11.97  10.04\n",
       "(javascript,)   17.68  23.01 17.83   28.14   16.89   27.68  26.35\n",
       "(python,)        5.05  11.04  6.41   15.98    6.94   15.60  12.31"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"javascript\",1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "463a191f",
   "metadata": {},
   "source": [
    "### MCMD<sub>2-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "41ea4761",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp)</th>\n",
       "      <td>5.89</td>\n",
       "      <td>11.51</td>\n",
       "      <td>7.00</td>\n",
       "      <td>15.94</td>\n",
       "      <td>7.14</td>\n",
       "      <td>15.58</td>\n",
       "      <td>13.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java)</th>\n",
       "      <td>6.32</td>\n",
       "      <td>11.90</td>\n",
       "      <td>7.25</td>\n",
       "      <td>16.44</td>\n",
       "      <td>7.50</td>\n",
       "      <td>16.06</td>\n",
       "      <td>13.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, javascript)</th>\n",
       "      <td>12.57</td>\n",
       "      <td>17.38</td>\n",
       "      <td>13.20</td>\n",
       "      <td>21.61</td>\n",
       "      <td>12.93</td>\n",
       "      <td>21.22</td>\n",
       "      <td>18.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, python)</th>\n",
       "      <td>6.06</td>\n",
       "      <td>11.58</td>\n",
       "      <td>7.25</td>\n",
       "      <td>16.31</td>\n",
       "      <td>7.49</td>\n",
       "      <td>15.92</td>\n",
       "      <td>12.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java)</th>\n",
       "      <td>4.59</td>\n",
       "      <td>10.28</td>\n",
       "      <td>5.81</td>\n",
       "      <td>14.31</td>\n",
       "      <td>6.02</td>\n",
       "      <td>13.95</td>\n",
       "      <td>11.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, javascript)</th>\n",
       "      <td>17.87</td>\n",
       "      <td>23.12</td>\n",
       "      <td>18.00</td>\n",
       "      <td>28.18</td>\n",
       "      <td>17.07</td>\n",
       "      <td>27.69</td>\n",
       "      <td>26.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, python)</th>\n",
       "      <td>5.24</td>\n",
       "      <td>11.22</td>\n",
       "      <td>6.55</td>\n",
       "      <td>16.13</td>\n",
       "      <td>7.01</td>\n",
       "      <td>15.70</td>\n",
       "      <td>12.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, javascript)</th>\n",
       "      <td>17.77</td>\n",
       "      <td>22.98</td>\n",
       "      <td>17.89</td>\n",
       "      <td>28.05</td>\n",
       "      <td>16.96</td>\n",
       "      <td>27.56</td>\n",
       "      <td>26.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, python)</th>\n",
       "      <td>5.58</td>\n",
       "      <td>11.25</td>\n",
       "      <td>6.78</td>\n",
       "      <td>15.94</td>\n",
       "      <td>6.92</td>\n",
       "      <td>15.49</td>\n",
       "      <td>12.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(javascript, python)</th>\n",
       "      <td>17.89</td>\n",
       "      <td>23.13</td>\n",
       "      <td>17.99</td>\n",
       "      <td>28.28</td>\n",
       "      <td>17.08</td>\n",
       "      <td>27.77</td>\n",
       "      <td>26.51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L Meteor\n",
       "(cpp, csharp)           5.89  11.51  7.00   15.94    7.14   15.58  13.01\n",
       "(cpp, java)             6.32  11.90  7.25   16.44    7.50   16.06  13.46\n",
       "(cpp, javascript)      12.57  17.38 13.20   21.61   12.93   21.22  18.86\n",
       "(cpp, python)           6.06  11.58  7.25   16.31    7.49   15.92  12.68\n",
       "(csharp, java)          4.59  10.28  5.81   14.31    6.02   13.95  11.43\n",
       "(csharp, javascript)   17.87  23.12 18.00   28.18   17.07   27.69  26.40\n",
       "(csharp, python)        5.24  11.22  6.55   16.13    7.01   15.70  12.51\n",
       "(java, javascript)     17.77  22.98 17.89   28.05   16.96   27.56  26.31\n",
       "(java, python)          5.58  11.25  6.78   15.94    6.92   15.49  12.77\n",
       "(javascript, python)   17.89  23.13 17.99   28.28   17.08   27.77  26.51"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"javascript\",2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9b1017b",
   "metadata": {},
   "source": [
    "### MCMD<sub>3-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e4d51055",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java)</th>\n",
       "      <td>6.22</td>\n",
       "      <td>11.91</td>\n",
       "      <td>7.19</td>\n",
       "      <td>16.51</td>\n",
       "      <td>7.51</td>\n",
       "      <td>16.10</td>\n",
       "      <td>13.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, javascript)</th>\n",
       "      <td>13.47</td>\n",
       "      <td>18.83</td>\n",
       "      <td>14.11</td>\n",
       "      <td>23.56</td>\n",
       "      <td>13.98</td>\n",
       "      <td>23.12</td>\n",
       "      <td>20.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, python)</th>\n",
       "      <td>6.32</td>\n",
       "      <td>12.03</td>\n",
       "      <td>7.59</td>\n",
       "      <td>16.36</td>\n",
       "      <td>7.65</td>\n",
       "      <td>15.94</td>\n",
       "      <td>13.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, javascript)</th>\n",
       "      <td>18.06</td>\n",
       "      <td>23.30</td>\n",
       "      <td>18.15</td>\n",
       "      <td>28.54</td>\n",
       "      <td>17.27</td>\n",
       "      <td>28.02</td>\n",
       "      <td>26.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, python)</th>\n",
       "      <td>6.83</td>\n",
       "      <td>12.53</td>\n",
       "      <td>7.89</td>\n",
       "      <td>17.61</td>\n",
       "      <td>7.98</td>\n",
       "      <td>17.12</td>\n",
       "      <td>14.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, javascript, python)</th>\n",
       "      <td>12.10</td>\n",
       "      <td>17.41</td>\n",
       "      <td>12.88</td>\n",
       "      <td>21.71</td>\n",
       "      <td>12.62</td>\n",
       "      <td>21.30</td>\n",
       "      <td>18.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, javascript)</th>\n",
       "      <td>18.06</td>\n",
       "      <td>23.20</td>\n",
       "      <td>18.13</td>\n",
       "      <td>28.40</td>\n",
       "      <td>17.20</td>\n",
       "      <td>27.89</td>\n",
       "      <td>26.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, python)</th>\n",
       "      <td>5.38</td>\n",
       "      <td>11.36</td>\n",
       "      <td>6.54</td>\n",
       "      <td>16.34</td>\n",
       "      <td>6.97</td>\n",
       "      <td>15.86</td>\n",
       "      <td>12.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, javascript, python)</th>\n",
       "      <td>17.96</td>\n",
       "      <td>23.31</td>\n",
       "      <td>18.10</td>\n",
       "      <td>28.48</td>\n",
       "      <td>17.19</td>\n",
       "      <td>27.96</td>\n",
       "      <td>26.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, javascript, python)</th>\n",
       "      <td>17.86</td>\n",
       "      <td>23.17</td>\n",
       "      <td>18.03</td>\n",
       "      <td>28.33</td>\n",
       "      <td>17.18</td>\n",
       "      <td>27.82</td>\n",
       "      <td>26.53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L  \\\n",
       "(cpp, csharp, java)             6.22  11.91  7.19   16.51    7.51   16.10   \n",
       "(cpp, csharp, javascript)      13.47  18.83 14.11   23.56   13.98   23.12   \n",
       "(cpp, csharp, python)           6.32  12.03  7.59   16.36    7.65   15.94   \n",
       "(cpp, java, javascript)        18.06  23.30 18.15   28.54   17.27   28.02   \n",
       "(cpp, java, python)             6.83  12.53  7.89   17.61    7.98   17.12   \n",
       "(cpp, javascript, python)      12.10  17.41 12.88   21.71   12.62   21.30   \n",
       "(csharp, java, javascript)     18.06  23.20 18.13   28.40   17.20   27.89   \n",
       "(csharp, java, python)          5.38  11.36  6.54   16.34    6.97   15.86   \n",
       "(csharp, javascript, python)   17.96  23.31 18.10   28.48   17.19   27.96   \n",
       "(java, javascript, python)     17.86  23.17 18.03   28.33   17.18   27.82   \n",
       "\n",
       "                             Meteor  \n",
       "(cpp, csharp, java)           13.35  \n",
       "(cpp, csharp, javascript)     20.26  \n",
       "(cpp, csharp, python)         13.06  \n",
       "(cpp, java, javascript)       26.72  \n",
       "(cpp, java, python)           14.48  \n",
       "(cpp, javascript, python)     18.50  \n",
       "(csharp, java, javascript)    26.65  \n",
       "(csharp, java, python)        12.67  \n",
       "(csharp, javascript, python)  26.66  \n",
       "(java, javascript, python)    26.53  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"javascript\",3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ed7b104",
   "metadata": {},
   "source": [
    "### MCMD<sub>4-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c514e260",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, javascript)</th>\n",
       "      <td>18.07</td>\n",
       "      <td>23.15</td>\n",
       "      <td>18.10</td>\n",
       "      <td>28.34</td>\n",
       "      <td>17.27</td>\n",
       "      <td>27.84</td>\n",
       "      <td>26.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, python)</th>\n",
       "      <td>6.14</td>\n",
       "      <td>12.05</td>\n",
       "      <td>7.47</td>\n",
       "      <td>16.45</td>\n",
       "      <td>7.63</td>\n",
       "      <td>15.99</td>\n",
       "      <td>12.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, javascript, python)</th>\n",
       "      <td>18.02</td>\n",
       "      <td>23.19</td>\n",
       "      <td>18.11</td>\n",
       "      <td>28.41</td>\n",
       "      <td>17.19</td>\n",
       "      <td>27.88</td>\n",
       "      <td>26.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, javascript, python)</th>\n",
       "      <td>18.02</td>\n",
       "      <td>23.28</td>\n",
       "      <td>18.09</td>\n",
       "      <td>28.57</td>\n",
       "      <td>17.21</td>\n",
       "      <td>28.04</td>\n",
       "      <td>26.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, javascript, python)</th>\n",
       "      <td>18.17</td>\n",
       "      <td>23.39</td>\n",
       "      <td>18.23</td>\n",
       "      <td>28.59</td>\n",
       "      <td>17.33</td>\n",
       "      <td>28.08</td>\n",
       "      <td>26.79</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   B-Moses B-Norm  B-CC Rouge-1 Rouge-2  \\\n",
       "(cpp, csharp, java, javascript)      18.07  23.15 18.10   28.34   17.27   \n",
       "(cpp, csharp, java, python)           6.14  12.05  7.47   16.45    7.63   \n",
       "(cpp, csharp, javascript, python)    18.02  23.19 18.11   28.41   17.19   \n",
       "(cpp, java, javascript, python)      18.02  23.28 18.09   28.57   17.21   \n",
       "(csharp, java, javascript, python)   18.17  23.39 18.23   28.59   17.33   \n",
       "\n",
       "                                   Rouge-L Meteor  \n",
       "(cpp, csharp, java, javascript)      27.84  26.53  \n",
       "(cpp, csharp, java, python)          15.99  12.96  \n",
       "(cpp, csharp, javascript, python)    27.88  26.62  \n",
       "(cpp, java, javascript, python)      28.04  26.69  \n",
       "(csharp, java, javascript, python)   28.08  26.79  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"javascript\",4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5dc3f342",
   "metadata": {},
   "source": [
    "### MCMD<sub>5-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "67bd4505",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, javascript, python)</th>\n",
       "      <td>18.13</td>\n",
       "      <td>23.37</td>\n",
       "      <td>18.26</td>\n",
       "      <td>28.62</td>\n",
       "      <td>17.37</td>\n",
       "      <td>28.07</td>\n",
       "      <td>26.82</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        B-Moses B-Norm  B-CC Rouge-1 Rouge-2  \\\n",
       "(cpp, csharp, java, javascript, python)   18.13  23.37 18.26   28.62   17.37   \n",
       "\n",
       "                                        Rouge-L Meteor  \n",
       "(cpp, csharp, java, javascript, python)   28.07  26.82  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"javascript\",5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dada4c44",
   "metadata": {},
   "source": [
    "## Test PL: C++"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bad16cc1",
   "metadata": {},
   "source": [
    "### MCMD<sub>1-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0012681a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp,)</th>\n",
       "      <td>10.58</td>\n",
       "      <td>17.09</td>\n",
       "      <td>12.02</td>\n",
       "      <td>21.62</td>\n",
       "      <td>11.71</td>\n",
       "      <td>21.25</td>\n",
       "      <td>19.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp,)</th>\n",
       "      <td>1.97</td>\n",
       "      <td>6.81</td>\n",
       "      <td>3.25</td>\n",
       "      <td>10.14</td>\n",
       "      <td>3.65</td>\n",
       "      <td>9.93</td>\n",
       "      <td>7.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java,)</th>\n",
       "      <td>1.98</td>\n",
       "      <td>6.70</td>\n",
       "      <td>3.23</td>\n",
       "      <td>9.40</td>\n",
       "      <td>3.05</td>\n",
       "      <td>9.14</td>\n",
       "      <td>7.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(javascript,)</th>\n",
       "      <td>1.67</td>\n",
       "      <td>6.82</td>\n",
       "      <td>2.84</td>\n",
       "      <td>10.36</td>\n",
       "      <td>3.19</td>\n",
       "      <td>10.11</td>\n",
       "      <td>7.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(python,)</th>\n",
       "      <td>1.67</td>\n",
       "      <td>7.36</td>\n",
       "      <td>3.04</td>\n",
       "      <td>11.38</td>\n",
       "      <td>3.80</td>\n",
       "      <td>11.08</td>\n",
       "      <td>7.64</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L Meteor\n",
       "(cpp,)          10.58  17.09 12.02   21.62   11.71   21.25  19.02\n",
       "(csharp,)        1.97   6.81  3.25   10.14    3.65    9.93   7.42\n",
       "(java,)          1.98   6.70  3.23    9.40    3.05    9.14   7.02\n",
       "(javascript,)    1.67   6.82  2.84   10.36    3.19   10.11   7.00\n",
       "(python,)        1.67   7.36  3.04   11.38    3.80   11.08   7.64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"cpp\",1) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b072de1e",
   "metadata": {},
   "source": [
    "### MCMD<sub>2-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d5cb8809",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp)</th>\n",
       "      <td>10.70</td>\n",
       "      <td>17.02</td>\n",
       "      <td>12.00</td>\n",
       "      <td>21.45</td>\n",
       "      <td>11.70</td>\n",
       "      <td>21.07</td>\n",
       "      <td>19.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java)</th>\n",
       "      <td>10.58</td>\n",
       "      <td>16.98</td>\n",
       "      <td>11.98</td>\n",
       "      <td>21.42</td>\n",
       "      <td>11.80</td>\n",
       "      <td>21.04</td>\n",
       "      <td>18.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, javascript)</th>\n",
       "      <td>7.31</td>\n",
       "      <td>12.57</td>\n",
       "      <td>8.23</td>\n",
       "      <td>16.15</td>\n",
       "      <td>8.74</td>\n",
       "      <td>15.82</td>\n",
       "      <td>12.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, python)</th>\n",
       "      <td>10.00</td>\n",
       "      <td>16.04</td>\n",
       "      <td>11.25</td>\n",
       "      <td>20.53</td>\n",
       "      <td>11.21</td>\n",
       "      <td>20.15</td>\n",
       "      <td>17.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java)</th>\n",
       "      <td>2.16</td>\n",
       "      <td>7.49</td>\n",
       "      <td>3.42</td>\n",
       "      <td>11.20</td>\n",
       "      <td>3.97</td>\n",
       "      <td>10.88</td>\n",
       "      <td>7.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, javascript)</th>\n",
       "      <td>1.95</td>\n",
       "      <td>7.48</td>\n",
       "      <td>3.20</td>\n",
       "      <td>11.60</td>\n",
       "      <td>4.11</td>\n",
       "      <td>11.28</td>\n",
       "      <td>7.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, python)</th>\n",
       "      <td>2.02</td>\n",
       "      <td>7.69</td>\n",
       "      <td>3.35</td>\n",
       "      <td>11.76</td>\n",
       "      <td>4.22</td>\n",
       "      <td>11.45</td>\n",
       "      <td>8.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, javascript)</th>\n",
       "      <td>2.05</td>\n",
       "      <td>7.39</td>\n",
       "      <td>3.29</td>\n",
       "      <td>11.16</td>\n",
       "      <td>3.82</td>\n",
       "      <td>10.81</td>\n",
       "      <td>7.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, python)</th>\n",
       "      <td>1.97</td>\n",
       "      <td>7.71</td>\n",
       "      <td>3.23</td>\n",
       "      <td>11.27</td>\n",
       "      <td>3.78</td>\n",
       "      <td>10.96</td>\n",
       "      <td>7.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(javascript, python)</th>\n",
       "      <td>1.85</td>\n",
       "      <td>7.88</td>\n",
       "      <td>3.18</td>\n",
       "      <td>12.23</td>\n",
       "      <td>4.12</td>\n",
       "      <td>11.85</td>\n",
       "      <td>8.04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L Meteor\n",
       "(cpp, csharp)          10.70  17.02 12.00   21.45   11.70   21.07  19.00\n",
       "(cpp, java)            10.58  16.98 11.98   21.42   11.80   21.04  18.90\n",
       "(cpp, javascript)       7.31  12.57  8.23   16.15    8.74   15.82  12.72\n",
       "(cpp, python)          10.00  16.04 11.25   20.53   11.21   20.15  17.46\n",
       "(csharp, java)          2.16   7.49  3.42   11.20    3.97   10.88   7.83\n",
       "(csharp, javascript)    1.95   7.48  3.20   11.60    4.11   11.28   7.68\n",
       "(csharp, python)        2.02   7.69  3.35   11.76    4.22   11.45   8.14\n",
       "(java, javascript)      2.05   7.39  3.29   11.16    3.82   10.81   7.56\n",
       "(java, python)          1.97   7.71  3.23   11.27    3.78   10.96   7.77\n",
       "(javascript, python)    1.85   7.88  3.18   12.23    4.12   11.85   8.04"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"cpp\",2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cef1aa5",
   "metadata": {},
   "source": [
    "### MCMD<sub>3-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "352d8857",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java)</th>\n",
       "      <td>10.84</td>\n",
       "      <td>17.09</td>\n",
       "      <td>12.17</td>\n",
       "      <td>21.59</td>\n",
       "      <td>11.86</td>\n",
       "      <td>21.21</td>\n",
       "      <td>19.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, javascript)</th>\n",
       "      <td>8.22</td>\n",
       "      <td>14.01</td>\n",
       "      <td>9.19</td>\n",
       "      <td>18.06</td>\n",
       "      <td>9.59</td>\n",
       "      <td>17.61</td>\n",
       "      <td>14.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, python)</th>\n",
       "      <td>8.49</td>\n",
       "      <td>14.47</td>\n",
       "      <td>9.47</td>\n",
       "      <td>18.59</td>\n",
       "      <td>9.62</td>\n",
       "      <td>18.14</td>\n",
       "      <td>15.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, javascript)</th>\n",
       "      <td>10.92</td>\n",
       "      <td>17.35</td>\n",
       "      <td>12.28</td>\n",
       "      <td>22.03</td>\n",
       "      <td>12.02</td>\n",
       "      <td>21.57</td>\n",
       "      <td>19.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, python)</th>\n",
       "      <td>10.98</td>\n",
       "      <td>17.34</td>\n",
       "      <td>12.35</td>\n",
       "      <td>22.04</td>\n",
       "      <td>12.10</td>\n",
       "      <td>21.61</td>\n",
       "      <td>19.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, javascript, python)</th>\n",
       "      <td>7.60</td>\n",
       "      <td>12.98</td>\n",
       "      <td>8.39</td>\n",
       "      <td>16.25</td>\n",
       "      <td>8.46</td>\n",
       "      <td>15.88</td>\n",
       "      <td>12.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, javascript)</th>\n",
       "      <td>2.17</td>\n",
       "      <td>7.91</td>\n",
       "      <td>3.45</td>\n",
       "      <td>12.08</td>\n",
       "      <td>4.27</td>\n",
       "      <td>11.67</td>\n",
       "      <td>8.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, python)</th>\n",
       "      <td>1.96</td>\n",
       "      <td>7.83</td>\n",
       "      <td>3.30</td>\n",
       "      <td>12.01</td>\n",
       "      <td>4.11</td>\n",
       "      <td>11.64</td>\n",
       "      <td>8.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, javascript, python)</th>\n",
       "      <td>2.22</td>\n",
       "      <td>8.17</td>\n",
       "      <td>3.51</td>\n",
       "      <td>12.41</td>\n",
       "      <td>4.41</td>\n",
       "      <td>12.02</td>\n",
       "      <td>8.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, javascript, python)</th>\n",
       "      <td>1.95</td>\n",
       "      <td>7.95</td>\n",
       "      <td>3.35</td>\n",
       "      <td>11.89</td>\n",
       "      <td>4.09</td>\n",
       "      <td>11.51</td>\n",
       "      <td>8.12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L  \\\n",
       "(cpp, csharp, java)            10.84  17.09 12.17   21.59   11.86   21.21   \n",
       "(cpp, csharp, javascript)       8.22  14.01  9.19   18.06    9.59   17.61   \n",
       "(cpp, csharp, python)           8.49  14.47  9.47   18.59    9.62   18.14   \n",
       "(cpp, java, javascript)        10.92  17.35 12.28   22.03   12.02   21.57   \n",
       "(cpp, java, python)            10.98  17.34 12.35   22.04   12.10   21.61   \n",
       "(cpp, javascript, python)       7.60  12.98  8.39   16.25    8.46   15.88   \n",
       "(csharp, java, javascript)      2.17   7.91  3.45   12.08    4.27   11.67   \n",
       "(csharp, java, python)          1.96   7.83  3.30   12.01    4.11   11.64   \n",
       "(csharp, javascript, python)    2.22   8.17  3.51   12.41    4.41   12.02   \n",
       "(java, javascript, python)      1.95   7.95  3.35   11.89    4.09   11.51   \n",
       "\n",
       "                             Meteor  \n",
       "(cpp, csharp, java)           19.19  \n",
       "(cpp, csharp, javascript)     14.42  \n",
       "(cpp, csharp, python)         15.13  \n",
       "(cpp, java, javascript)       19.55  \n",
       "(cpp, java, python)           19.63  \n",
       "(cpp, javascript, python)     12.89  \n",
       "(csharp, java, javascript)     8.12  \n",
       "(csharp, java, python)         8.04  \n",
       "(csharp, javascript, python)   8.44  \n",
       "(java, javascript, python)     8.12  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"cpp\",3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8322e37e",
   "metadata": {},
   "source": [
    "### MCMD<sub>4-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "45da1ef9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, javascript)</th>\n",
       "      <td>11.08</td>\n",
       "      <td>17.23</td>\n",
       "      <td>12.39</td>\n",
       "      <td>21.85</td>\n",
       "      <td>12.09</td>\n",
       "      <td>21.39</td>\n",
       "      <td>19.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, python)</th>\n",
       "      <td>8.42</td>\n",
       "      <td>14.48</td>\n",
       "      <td>9.43</td>\n",
       "      <td>18.62</td>\n",
       "      <td>9.70</td>\n",
       "      <td>18.16</td>\n",
       "      <td>15.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, javascript, python)</th>\n",
       "      <td>11.04</td>\n",
       "      <td>17.52</td>\n",
       "      <td>12.38</td>\n",
       "      <td>22.22</td>\n",
       "      <td>12.10</td>\n",
       "      <td>21.73</td>\n",
       "      <td>19.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, javascript, python)</th>\n",
       "      <td>11.18</td>\n",
       "      <td>17.48</td>\n",
       "      <td>12.50</td>\n",
       "      <td>22.21</td>\n",
       "      <td>12.19</td>\n",
       "      <td>21.76</td>\n",
       "      <td>19.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, javascript, python)</th>\n",
       "      <td>2.35</td>\n",
       "      <td>8.26</td>\n",
       "      <td>3.66</td>\n",
       "      <td>12.67</td>\n",
       "      <td>4.46</td>\n",
       "      <td>12.23</td>\n",
       "      <td>8.72</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   B-Moses B-Norm  B-CC Rouge-1 Rouge-2  \\\n",
       "(cpp, csharp, java, javascript)      11.08  17.23 12.39   21.85   12.09   \n",
       "(cpp, csharp, java, python)           8.42  14.48  9.43   18.62    9.70   \n",
       "(cpp, csharp, javascript, python)    11.04  17.52 12.38   22.22   12.10   \n",
       "(cpp, java, javascript, python)      11.18  17.48 12.50   22.21   12.19   \n",
       "(csharp, java, javascript, python)    2.35   8.26  3.66   12.67    4.46   \n",
       "\n",
       "                                   Rouge-L Meteor  \n",
       "(cpp, csharp, java, javascript)      21.39  19.50  \n",
       "(cpp, csharp, java, python)          18.16  15.09  \n",
       "(cpp, csharp, javascript, python)    21.73  19.65  \n",
       "(cpp, java, javascript, python)      21.76  19.74  \n",
       "(csharp, java, javascript, python)   12.23   8.72  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"cpp\",4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e19ccb16",
   "metadata": {},
   "source": [
    "### MCMD<sub>5-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "aed1775b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, javascript, python)</th>\n",
       "      <td>11.14</td>\n",
       "      <td>17.61</td>\n",
       "      <td>12.51</td>\n",
       "      <td>22.43</td>\n",
       "      <td>12.29</td>\n",
       "      <td>21.97</td>\n",
       "      <td>19.79</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        B-Moses B-Norm  B-CC Rouge-1 Rouge-2  \\\n",
       "(cpp, csharp, java, javascript, python)   11.14  17.61 12.51   22.43   12.29   \n",
       "\n",
       "                                        Rouge-L Meteor  \n",
       "(cpp, csharp, java, javascript, python)   21.97  19.79  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"cpp\",5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae859afa",
   "metadata": {},
   "source": [
    "## Test PL: `C#`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1bec2924",
   "metadata": {},
   "source": [
    "### MCMD<sub>1-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9e93d1e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp,)</th>\n",
       "      <td>1.94</td>\n",
       "      <td>7.26</td>\n",
       "      <td>3.39</td>\n",
       "      <td>9.17</td>\n",
       "      <td>3.39</td>\n",
       "      <td>9.02</td>\n",
       "      <td>7.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp,)</th>\n",
       "      <td>24.25</td>\n",
       "      <td>24.03</td>\n",
       "      <td>20.28</td>\n",
       "      <td>27.31</td>\n",
       "      <td>19.13</td>\n",
       "      <td>27.03</td>\n",
       "      <td>26.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java,)</th>\n",
       "      <td>1.60</td>\n",
       "      <td>6.11</td>\n",
       "      <td>2.92</td>\n",
       "      <td>7.28</td>\n",
       "      <td>2.32</td>\n",
       "      <td>7.16</td>\n",
       "      <td>6.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(javascript,)</th>\n",
       "      <td>1.48</td>\n",
       "      <td>6.52</td>\n",
       "      <td>2.72</td>\n",
       "      <td>8.91</td>\n",
       "      <td>2.57</td>\n",
       "      <td>8.76</td>\n",
       "      <td>6.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(python,)</th>\n",
       "      <td>1.65</td>\n",
       "      <td>6.85</td>\n",
       "      <td>2.92</td>\n",
       "      <td>9.24</td>\n",
       "      <td>3.20</td>\n",
       "      <td>9.08</td>\n",
       "      <td>7.42</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L Meteor\n",
       "(cpp,)           1.94   7.26  3.39    9.17    3.39    9.02   7.47\n",
       "(csharp,)       24.25  24.03 20.28   27.31   19.13   27.03  26.01\n",
       "(java,)          1.60   6.11  2.92    7.28    2.32    7.16   6.37\n",
       "(javascript,)    1.48   6.52  2.72    8.91    2.57    8.76   6.93\n",
       "(python,)        1.65   6.85  2.92    9.24    3.20    9.08   7.42"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"csharp\",1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b864a6ec",
   "metadata": {},
   "source": [
    "### MCMD<sub>2-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "1aa56c1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp)</th>\n",
       "      <td>23.86</td>\n",
       "      <td>23.65</td>\n",
       "      <td>19.94</td>\n",
       "      <td>26.85</td>\n",
       "      <td>18.79</td>\n",
       "      <td>26.56</td>\n",
       "      <td>25.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java)</th>\n",
       "      <td>2.49</td>\n",
       "      <td>7.48</td>\n",
       "      <td>3.80</td>\n",
       "      <td>9.51</td>\n",
       "      <td>3.68</td>\n",
       "      <td>9.35</td>\n",
       "      <td>7.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, javascript)</th>\n",
       "      <td>2.73</td>\n",
       "      <td>7.37</td>\n",
       "      <td>3.96</td>\n",
       "      <td>9.00</td>\n",
       "      <td>3.72</td>\n",
       "      <td>8.84</td>\n",
       "      <td>7.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, python)</th>\n",
       "      <td>2.57</td>\n",
       "      <td>7.59</td>\n",
       "      <td>3.95</td>\n",
       "      <td>9.59</td>\n",
       "      <td>3.98</td>\n",
       "      <td>9.42</td>\n",
       "      <td>7.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java)</th>\n",
       "      <td>24.03</td>\n",
       "      <td>23.97</td>\n",
       "      <td>20.12</td>\n",
       "      <td>27.21</td>\n",
       "      <td>18.94</td>\n",
       "      <td>26.91</td>\n",
       "      <td>25.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, javascript)</th>\n",
       "      <td>24.28</td>\n",
       "      <td>24.32</td>\n",
       "      <td>20.41</td>\n",
       "      <td>27.67</td>\n",
       "      <td>19.24</td>\n",
       "      <td>27.35</td>\n",
       "      <td>26.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, python)</th>\n",
       "      <td>24.20</td>\n",
       "      <td>24.27</td>\n",
       "      <td>20.30</td>\n",
       "      <td>27.50</td>\n",
       "      <td>19.08</td>\n",
       "      <td>27.20</td>\n",
       "      <td>26.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, javascript)</th>\n",
       "      <td>2.06</td>\n",
       "      <td>6.87</td>\n",
       "      <td>3.24</td>\n",
       "      <td>8.88</td>\n",
       "      <td>3.05</td>\n",
       "      <td>8.72</td>\n",
       "      <td>7.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, python)</th>\n",
       "      <td>2.12</td>\n",
       "      <td>6.87</td>\n",
       "      <td>3.33</td>\n",
       "      <td>8.93</td>\n",
       "      <td>3.29</td>\n",
       "      <td>8.76</td>\n",
       "      <td>7.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(javascript, python)</th>\n",
       "      <td>2.18</td>\n",
       "      <td>7.97</td>\n",
       "      <td>3.49</td>\n",
       "      <td>10.71</td>\n",
       "      <td>3.79</td>\n",
       "      <td>10.51</td>\n",
       "      <td>8.32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L Meteor\n",
       "(cpp, csharp)          23.86  23.65 19.94   26.85   18.79   26.56  25.68\n",
       "(cpp, java)             2.49   7.48  3.80    9.51    3.68    9.35   7.98\n",
       "(cpp, javascript)       2.73   7.37  3.96    9.00    3.72    8.84   7.59\n",
       "(cpp, python)           2.57   7.59  3.95    9.59    3.98    9.42   7.80\n",
       "(csharp, java)         24.03  23.97 20.12   27.21   18.94   26.91  25.83\n",
       "(csharp, javascript)   24.28  24.32 20.41   27.67   19.24   27.35  26.28\n",
       "(csharp, python)       24.20  24.27 20.30   27.50   19.08   27.20  26.21\n",
       "(java, javascript)      2.06   6.87  3.24    8.88    3.05    8.72   7.25\n",
       "(java, python)          2.12   6.87  3.33    8.93    3.29    8.76   7.35\n",
       "(javascript, python)    2.18   7.97  3.49   10.71    3.79   10.51   8.32"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"csharp\",2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5132f0a6",
   "metadata": {},
   "source": [
    "### MCMD<sub>3-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "11070824",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java)</th>\n",
       "      <td>23.99</td>\n",
       "      <td>23.87</td>\n",
       "      <td>20.05</td>\n",
       "      <td>27.10</td>\n",
       "      <td>18.86</td>\n",
       "      <td>26.79</td>\n",
       "      <td>25.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, javascript)</th>\n",
       "      <td>21.03</td>\n",
       "      <td>20.59</td>\n",
       "      <td>17.09</td>\n",
       "      <td>23.71</td>\n",
       "      <td>16.51</td>\n",
       "      <td>23.44</td>\n",
       "      <td>21.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, python)</th>\n",
       "      <td>21.42</td>\n",
       "      <td>21.24</td>\n",
       "      <td>17.56</td>\n",
       "      <td>24.45</td>\n",
       "      <td>16.91</td>\n",
       "      <td>24.15</td>\n",
       "      <td>22.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, javascript)</th>\n",
       "      <td>2.66</td>\n",
       "      <td>7.90</td>\n",
       "      <td>3.89</td>\n",
       "      <td>10.16</td>\n",
       "      <td>3.75</td>\n",
       "      <td>9.95</td>\n",
       "      <td>8.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, python)</th>\n",
       "      <td>2.89</td>\n",
       "      <td>8.05</td>\n",
       "      <td>4.13</td>\n",
       "      <td>10.43</td>\n",
       "      <td>4.20</td>\n",
       "      <td>10.25</td>\n",
       "      <td>8.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, javascript, python)</th>\n",
       "      <td>3.11</td>\n",
       "      <td>8.36</td>\n",
       "      <td>4.58</td>\n",
       "      <td>9.66</td>\n",
       "      <td>4.10</td>\n",
       "      <td>9.49</td>\n",
       "      <td>8.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, javascript)</th>\n",
       "      <td>24.33</td>\n",
       "      <td>24.31</td>\n",
       "      <td>20.38</td>\n",
       "      <td>27.58</td>\n",
       "      <td>19.17</td>\n",
       "      <td>27.26</td>\n",
       "      <td>26.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, python)</th>\n",
       "      <td>23.78</td>\n",
       "      <td>23.84</td>\n",
       "      <td>19.91</td>\n",
       "      <td>27.13</td>\n",
       "      <td>18.81</td>\n",
       "      <td>26.82</td>\n",
       "      <td>25.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, javascript, python)</th>\n",
       "      <td>24.41</td>\n",
       "      <td>24.42</td>\n",
       "      <td>20.47</td>\n",
       "      <td>27.70</td>\n",
       "      <td>19.27</td>\n",
       "      <td>27.38</td>\n",
       "      <td>26.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(java, javascript, python)</th>\n",
       "      <td>2.08</td>\n",
       "      <td>7.39</td>\n",
       "      <td>3.39</td>\n",
       "      <td>9.76</td>\n",
       "      <td>3.57</td>\n",
       "      <td>9.59</td>\n",
       "      <td>7.66</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             B-Moses B-Norm  B-CC Rouge-1 Rouge-2 Rouge-L  \\\n",
       "(cpp, csharp, java)            23.99  23.87 20.05   27.10   18.86   26.79   \n",
       "(cpp, csharp, javascript)      21.03  20.59 17.09   23.71   16.51   23.44   \n",
       "(cpp, csharp, python)          21.42  21.24 17.56   24.45   16.91   24.15   \n",
       "(cpp, java, javascript)         2.66   7.90  3.89   10.16    3.75    9.95   \n",
       "(cpp, java, python)             2.89   8.05  4.13   10.43    4.20   10.25   \n",
       "(cpp, javascript, python)       3.11   8.36  4.58    9.66    4.10    9.49   \n",
       "(csharp, java, javascript)     24.33  24.31 20.38   27.58   19.17   27.26   \n",
       "(csharp, java, python)         23.78  23.84 19.91   27.13   18.81   26.82   \n",
       "(csharp, javascript, python)   24.41  24.42 20.47   27.70   19.27   27.38   \n",
       "(java, javascript, python)      2.08   7.39  3.39    9.76    3.57    9.59   \n",
       "\n",
       "                             Meteor  \n",
       "(cpp, csharp, java)           25.88  \n",
       "(cpp, csharp, javascript)     21.46  \n",
       "(cpp, csharp, python)         22.27  \n",
       "(cpp, java, javascript)        8.43  \n",
       "(cpp, java, python)            8.69  \n",
       "(cpp, javascript, python)      8.14  \n",
       "(csharp, java, javascript)    26.25  \n",
       "(csharp, java, python)        25.77  \n",
       "(csharp, javascript, python)  26.42  \n",
       "(java, javascript, python)     7.66  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"csharp\",3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b72b14d9",
   "metadata": {},
   "source": [
    "### MCMD<sub>4-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d84efd02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, javascript)</th>\n",
       "      <td>24.13</td>\n",
       "      <td>23.95</td>\n",
       "      <td>20.19</td>\n",
       "      <td>27.22</td>\n",
       "      <td>18.98</td>\n",
       "      <td>26.90</td>\n",
       "      <td>25.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, python)</th>\n",
       "      <td>21.60</td>\n",
       "      <td>21.12</td>\n",
       "      <td>17.55</td>\n",
       "      <td>24.50</td>\n",
       "      <td>17.00</td>\n",
       "      <td>24.19</td>\n",
       "      <td>22.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, javascript, python)</th>\n",
       "      <td>24.03</td>\n",
       "      <td>24.10</td>\n",
       "      <td>20.14</td>\n",
       "      <td>27.46</td>\n",
       "      <td>19.03</td>\n",
       "      <td>27.14</td>\n",
       "      <td>26.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(cpp, java, javascript, python)</th>\n",
       "      <td>3.03</td>\n",
       "      <td>8.23</td>\n",
       "      <td>4.24</td>\n",
       "      <td>10.70</td>\n",
       "      <td>4.15</td>\n",
       "      <td>10.48</td>\n",
       "      <td>8.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(csharp, java, javascript, python)</th>\n",
       "      <td>24.09</td>\n",
       "      <td>24.15</td>\n",
       "      <td>20.22</td>\n",
       "      <td>27.48</td>\n",
       "      <td>19.06</td>\n",
       "      <td>27.16</td>\n",
       "      <td>26.09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   B-Moses B-Norm  B-CC Rouge-1 Rouge-2  \\\n",
       "(cpp, csharp, java, javascript)      24.13  23.95 20.19   27.22   18.98   \n",
       "(cpp, csharp, java, python)          21.60  21.12 17.55   24.50   17.00   \n",
       "(cpp, csharp, javascript, python)    24.03  24.10 20.14   27.46   19.03   \n",
       "(cpp, java, javascript, python)       3.03   8.23  4.24   10.70    4.15   \n",
       "(csharp, java, javascript, python)   24.09  24.15 20.22   27.48   19.06   \n",
       "\n",
       "                                   Rouge-L Meteor  \n",
       "(cpp, csharp, java, javascript)      26.90  25.95  \n",
       "(cpp, csharp, java, python)          24.19  22.44  \n",
       "(cpp, csharp, javascript, python)    27.14  26.08  \n",
       "(cpp, java, javascript, python)      10.48   8.83  \n",
       "(csharp, java, javascript, python)   27.16  26.09  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"csharp\",4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e18520be",
   "metadata": {},
   "source": [
    "### MCMD<sub>5-PL</sub>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2889fa76",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "      <th>Rouge-1</th>\n",
       "      <th>Rouge-2</th>\n",
       "      <th>Rouge-L</th>\n",
       "      <th>Meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(cpp, csharp, java, javascript, python)</th>\n",
       "      <td>24.19</td>\n",
       "      <td>24.29</td>\n",
       "      <td>20.33</td>\n",
       "      <td>27.70</td>\n",
       "      <td>19.20</td>\n",
       "      <td>27.36</td>\n",
       "      <td>26.27</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        B-Moses B-Norm  B-CC Rouge-1 Rouge-2  \\\n",
       "(cpp, csharp, java, javascript, python)   24.19  24.29 20.33   27.70   19.20   \n",
       "\n",
       "                                        Rouge-L Meteor  \n",
       "(cpp, csharp, java, javascript, python)   27.36  26.27  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_finetune(\"csharp\",5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:MCMD]",
   "language": "python",
   "name": "conda-env-MCMD-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}