llm-evaluation-metrics.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":8564359,"sourceType":"datasetVersion","datasetId":5120000},{"sourceId":8596421,"sourceType":"datasetVersion","datasetId":5081117},{"sourceId":8597947,"sourceType":"datasetVersion","datasetId":5093115},{"sourceId":8606226,"sourceType":"datasetVersion","datasetId":5149721},{"sourceId":8607239,"sourceType":"datasetVersion","datasetId":5150443},{"sourceId":8646487,"sourceType":"datasetVersion","datasetId":5178732},{"sourceId":8649522,"sourceType":"datasetVersion","datasetId":5149372},{"sourceId":8649921,"sourceType":"datasetVersion","datasetId":5180690},{"sourceId":8665536,"sourceType":"datasetVersion","datasetId":5192567}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U nltk\n!pip install nltk==3.5\n!pip install langchain\n!pip install rouge_score\n!pip install bert_score","metadata":{"execution":{"iopub.status.busy":"2024-06-11T14:10:18.940186Z","iopub.execute_input":"2024-06-11T14:10:18.941554Z","iopub.status.idle":"2024-06-11T14:11:39.389769Z","shell.execute_reply.started":"2024-06-11T14:10:18.941509Z","shell.execute_reply":"2024-06-11T14:11:39.38806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#this cell solves a problem with unzipping corpora/wordnet\n!unzip /usr/share/nltk_data/corpora/wordnet.zip -d /usr/share/nltk_data/corpora/","metadata":{"execution":{"iopub.status.busy":"2024-06-11T14:12:20.490391Z","iopub.execute_input":"2024-06-11T14:12:20.490869Z","iopub.status.idle":"2024-06-11T14:12:26.898008Z","shell.execute_reply.started":"2024-06-11T14:12:20.490835Z","shell.execute_reply":"2024-06-11T14:12:26.89629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nfrom langchain.evaluation import ExactMatchStringEvaluator\nimport nltk\nfrom nltk.translate.meteor_score import meteor_score as meteor\nfrom rouge_score import rouge_scorer\nfrom bert_score import score as bert_score\n\n# Initialize the ExactMatchStringEvaluator\nevaluator = ExactMatchStringEvaluator(\n    ignore_case=True,\n    ignore_punctuation=True,\n)\n\n# Initialize ROUGE scorer\nrouge = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)\n\nwith open('add-file-path', 'r') as f:\n    data = json.load(f)\n\n# Initialize the results\nexact_match_result = 0\ntotal_meteor_score = 0\ntotal_rouge1_score = 0\ntotal_rouge2_score = 0\ntotal_rougeL_score = 0\nall_responses = []\nall_answers = []\n\n# Iterate through the data and compute the metrics\nfor question, (response, answer) in data.items():\n    # Calculate Exact Match\n    ex = float(evaluator.evaluate_strings(prediction=response.strip(), reference=answer.strip())['score'])\n    exact_match_result += ex\n    \n    meteor_scr = meteor([answer], response)\n    total_meteor_score += meteor_scr\n    \n    # Calculate ROUGE\n    rouge_scores = rouge.score(response, answer)\n    total_rouge1_score += rouge_scores['rouge1'].fmeasure\n    total_rouge2_score += rouge_scores['rouge2'].fmeasure\n    total_rougeL_score += rouge_scores['rougeL'].fmeasure\n    \n    # Collect responses and answers for BERTScore\n    all_responses.append(response)\n    all_answers.append(answer)\n\n# Calculate BERTScore for all responses and answers\nP, R, F1 = bert_score(all_responses, all_answers, lang=\"en\")\n\n# Aggregate BERTScore results\ntotal_bert_precision = sum(P.tolist())\ntotal_bert_recall = sum(R.tolist())\ntotal_bert_f1 = sum(F1.tolist())\n\n# Final results\nnum_samples = len(data)\nexact_match_result /= num_samples\ntotal_meteor_score /= num_samples\ntotal_rouge1_score /= num_samples\ntotal_rouge2_score /= num_samples\ntotal_rougeL_score /= num_samples\ntotal_bert_precision /= num_samples\ntotal_bert_recall /= num_samples\ntotal_bert_f1 /= num_samples\n\n# Print the results\nprint(\"oracle_adore_1_few_1_esit.json\")\nprint(f\"Exact Match: {exact_match_result}\")\nprint(f\"METEOR: {total_meteor_score}\")\nprint(f\"ROUGE-1: {total_rouge1_score}\")\nprint(f\"ROUGE-2: {total_rouge2_score}\")\nprint(f\"ROUGE-L: {total_rougeL_score}\")\nprint(f\"BERT Precision: {total_bert_precision}\")\nprint(f\"BERT Recall: {total_bert_recall}\")\nprint(f\"BERT F1: {total_bert_f1}\")\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-11T14:12:08.196558Z","iopub.execute_input":"2024-06-11T14:12:08.197039Z","iopub.status.idle":"2024-06-11T14:12:08.647671Z","shell.execute_reply.started":"2024-06-11T14:12:08.196997Z","shell.execute_reply":"2024-06-11T14:12:08.645892Z"},"trusted":true},"execution_count":null,"outputs":[]}]}