Experiments with Re-Ranker.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9069c921",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import tokenizers\n",
    "import puz\n",
    "import os\n",
    "import numpy as np\n",
    "import streamlit as st\n",
    "import scipy\n",
    "import sys\n",
    "import subprocess\n",
    "import copy\n",
    "import json\n",
    "import pickle\n",
    "import shutil\n",
    "import time\n",
    "import regex\n",
    "import requests\n",
    "import datetime\n",
    "\n",
    "from itertools import zip_longest\n",
    "from copy import deepcopy\n",
    "from tqdm import tqdm\n",
    "from pprint import pprint\n",
    "from utils import puz_to_json\n",
    "\n",
    "from transformers import AutoTokenizer, T5ForConditionalGeneration\n",
    "from solver.Crossword import Crossword\n",
    "from solver.BPSolver import BPSolver\n",
    "from solver.Utils import print_grid\n",
    "from models import setup_closedbook, setup_t5_reranker, DPRForCrossword"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4c27c767",
   "metadata": {},
   "outputs": [],
   "source": [
    "def solve(crossword, max_candidate):\n",
    "    solver = BPSolver(crossword, max_candidates = max_candidate)\n",
    "    solution = solver.solve(num_iters = 60, iterative_improvement_steps = 10)\n",
    "    solver.evaluate(solution)\n",
    "    \n",
    "def json_CA_json_converter(json_file_path, is_path):\n",
    "    try:\n",
    "        if is_path:\n",
    "            with open(json_file_path, \"r\") as file:\n",
    "                data = json.load(file)\n",
    "        else:\n",
    "            data = json_file_path\n",
    "\n",
    "        json_conversion_dict = {}\n",
    "\n",
    "        rows = data[\"size\"][\"rows\"]\n",
    "        cols = data[\"size\"][\"cols\"]\n",
    "        date = data[\"date\"]\n",
    "\n",
    "        clues = data[\"clues\"]\n",
    "        answers = data[\"answers\"]\n",
    "\n",
    "        json_conversion_dict[\"metadata\"] = {\"date\": date, \"rows\": rows, \"cols\": cols}\n",
    "\n",
    "        across_clue_answer = {}\n",
    "        down_clue_answer = {}\n",
    "\n",
    "        for clue, ans in zip(clues[\"across\"], answers[\"across\"]):\n",
    "            split_clue = clue.split(\" \")\n",
    "            clue_num = split_clue[0][:-1]\n",
    "            clue_ = \" \".join(split_clue[1:])\n",
    "            clue_ = clue_.replace(\"[\", \"\").replace(\"]\", \"\")\n",
    "            across_clue_answer[clue_num] = [clue_, ans]\n",
    "\n",
    "        for clue, ans in zip(clues[\"down\"], answers[\"down\"]):\n",
    "            split_clue = clue.split(\" \")\n",
    "            clue_num = split_clue[0][:-1]\n",
    "            clue_ = \" \".join(split_clue[1:])\n",
    "            clue_ = clue_.replace(\"[\", \"\").replace(\"]\", \"\")\n",
    "            down_clue_answer[clue_num] = [clue_, ans]\n",
    "\n",
    "        json_conversion_dict[\"clues\"] = {\n",
    "            \"across\": across_clue_answer,\n",
    "            \"down\": down_clue_answer,\n",
    "        }\n",
    "\n",
    "        grid_info = data[\"grid\"]\n",
    "        grid_num = data[\"gridnums\"]\n",
    "\n",
    "        grid_info_list = []\n",
    "        for i in range(rows):\n",
    "            row_list = []\n",
    "            for j in range(cols):\n",
    "                if grid_info[i * rows + j] == \".\":\n",
    "                    row_list.append(\"BLACK\")\n",
    "                else:\n",
    "                    if grid_num[i * rows + j] == 0:\n",
    "                        row_list.append([\"\", grid_info[i * rows + j]])\n",
    "                    else:\n",
    "                        row_list.append(\n",
    "                            [str(grid_num[i * rows + j]), grid_info[i * rows + j]]\n",
    "                        )\n",
    "            grid_info_list.append(row_list)\n",
    "\n",
    "        json_conversion_dict[\"grid\"] = grid_info_list\n",
    "\n",
    "        return json_conversion_dict\n",
    "    \n",
    "    except:\n",
    "        print(\"ERROR has occured.\")\n",
    "\n",
    "def getGrid(dateStr):\n",
    "\n",
    "    headers = {\n",
    "        'Referer': 'https://www.xwordinfo.com/JSON/'\n",
    "    }\n",
    "    # mm/dd/yyyy\n",
    "\n",
    "    url = 'https://www.xwordinfo.com/JSON/Data.ashx?date=' + dateStr\n",
    "\n",
    "    response = requests.get(url, headers=headers)\n",
    "\n",
    "    context = {}\n",
    "    grid_data = {}\n",
    "    if response.status_code == 200:\n",
    "        bytevalue = response.content\n",
    "        jsonText = bytevalue.decode('utf-8').replace(\"'\", '\"')\n",
    "        grid_data = json.loads(jsonText)\n",
    "        puzzle_data = json_CA_json_converter(grid_data, False)\n",
    "        for dim in ['across', 'down']:\n",
    "            for grid_num in puzzle_data['clues'][dim].keys():\n",
    "                clue_answer_list = puzzle_data['clues'][dim][grid_num]\n",
    "                clue_section = clue_answer_list[0]\n",
    "                ans_section = clue_answer_list[1]\n",
    "                clue_section = clue_section.replace(\"&quot;\", \"'\").replace(\"&#39;\", \"'\")\n",
    "                puzzle_data['clues'][dim][grid_num] = [clue_section, ans_section]\n",
    "        return puzzle_data\n",
    "\n",
    "    else:\n",
    "        print(f\"Request failed with status code {response.status_code}.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "08548956",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "08/05/2023\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Overriding args parameter value from checkpoint state. Param = do_lower_case, value = True\n",
      "Overriding args parameter value from checkpoint state. Param = pretrained_model_cfg, value = distilbert-base-uncased\n",
      "Overriding args parameter value from checkpoint state. Param = encoder_model_type, value = hf_distilbert\n",
      "Overriding args parameter value from checkpoint state. Param = sequence_length, value = 32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "beginning BP iterations\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60/60 [00:17<00:00,  3.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done BP iterations\n",
      "\n",
      "Before II with T5-small: Letters Correct: 95.38461538461539% | Words Correct: 85.71428571428571%\n",
      "1th iteration: Letters Correct: 95.38461538461539% | Words Correct: 85.71428571428571%\n",
      "2th iteration: Letters Correct: 94.87179487179486% | Words Correct: 82.85714285714286%\n",
      "3th iteration: Letters Correct: 95.8974358974359% | Words Correct: 85.71428571428571%\n",
      "4th iteration: Letters Correct: 95.8974358974359% | Words Correct: 85.71428571428571%\n",
      "\n",
      "After II with ByT5: Letters Correct: 95.8974358974359% | Words Correct: 85.71428571428571%\n",
      "Letters Correct: 187/195 | Words Correct: 60/70\n",
      "Letters Correct: 95.8974358974359% | Words Correct: 85.71428571428571%\n",
      "Total time taken:  280.78948950767517\n"
     ]
    }
   ],
   "source": [
    "date_list = [datetime.date(2023, 8, 5)]\n",
    "\n",
    "for date in date_list:\n",
    "    formatted_date = date.strftime('%m/%d/%Y')\n",
    "    print(formatted_date)\n",
    "\n",
    "#     try:\n",
    "    puzzle = getGrid(formatted_date)\n",
    "    start_time = time.time()\n",
    "    crossword = Crossword(puzzle)\n",
    "    solve(crossword, 40000)\n",
    "    end_time = time.time()\n",
    "    print(\"Total time taken: \", end_time - start_time)\n",
    "#     except:\n",
    "#         print(\"Error Occured for date: \", formatted_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f54255c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import wordsegment\n",
    "import math\n",
    "from wordsegment import load, segment, clean\n",
    "load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbede151",
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary = set([a.strip() for a in open('solver/words_alpha.txt','r').readlines()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9b79fec",
   "metadata": {},
   "outputs": [],
   "source": [
    "def num_words(fill):\n",
    "    '''segment the text into multiple words and count how many words the text has in total'''\n",
    "    segmented = segment(fill)\n",
    "    prob = 0.0\n",
    "    for word in segmented:\n",
    "        if word not in dictionary:\n",
    "            return 999, -9999999999999\n",
    "        prob += math.log(wordsegment.UNIGRAMS[word])\n",
    "    return (len(segmented), prob)\n",
    "\n",
    "def get_word_flips(fill, num_candidates = 10):\n",
    "    results = {}\n",
    "    min_length = 999\n",
    "    fill = clean(fill)\n",
    "    for index, char in enumerate(fill):\n",
    "        for new_letter in 'abcdefghijklmnopqrstuvwxyz':\n",
    "            # place a letter \n",
    "            new_fill = list(fill)\n",
    "            new_fill[index] = new_letter\n",
    "            new_fill = ''.join(new_fill)\n",
    "            \n",
    "            curr_num_words, prob = num_words(new_fill)\n",
    "\n",
    "            if curr_num_words not in results:\n",
    "                results[curr_num_words] = []\n",
    "            results[curr_num_words].append((new_fill, prob))\n",
    "            if curr_num_words < min_length:\n",
    "                min_length = curr_num_words\n",
    "    if min_length == 999:\n",
    "        return [fill.upper()]\n",
    "    all_results = sum([sorted(results[length], key=lambda x:-x[1]) for length in sorted(list(results.keys()))], [])\n",
    "    print(len(all_results))\n",
    "    return [a[0].upper() for a in all_results[0:num_candidates]]\n",
    "\n",
    "def clean(text):\n",
    "    '''\n",
    "    :param text: question or answer text\n",
    "    :return: text with line breaks and trailing spaces removed\n",
    "    '''\n",
    "    return \" \".join(text.strip().split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0cdd710",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(get_word_flips(\"iluveyou\", 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bc3ce86",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5d6d21d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e62ac106",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2ddf51a",
   "metadata": {},
   "outputs": [],
   "source": [
    "crossword.letter_grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f772ee8",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_str = \"Letters Correct: 98.46153846153847% | Words Correct: 94.11764705882352%\"\n",
    "pattern = r\"\\d+\\.\\d+\"\n",
    "matches = re.findall(pattern, test_str)\n",
    "float_numbers = [float(match) for match in matches]\n",
    "print(float_numbers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab8c418f",
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 0\n",
    "for k, value in crossword.variables.items():\n",
    "    print(value['gold'])\n",
    "    counter += 1\n",
    "print(counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38317e83",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_clues = [\"'Try not to attract attention'\", 'Disney channel', \"'The Nutcracker' role\", 'One pulling some strings', 'Long part of a bouzouki', 'Big deals', \"Amy who wrote 'The Kitchen God's Wife'\", 'Played out', 'Dig locale', 'Not here', 'The Eagle, e.g., for short', 'Geographical inspiration for Strauss', \"'100% correct!'\", 'Safaris, e.g.', 'Not std.', 'Fish also known as wahoo', 'Border region along the Rhine', 'Bit of assistance', 'Run like a mouse', 'Defiant declaration popularized by the drag queen Bianca Del Rio', 'Shell-inspired shade of greenish blue', 'This and that', 'First Nigerian-born singer to win a Grammy', 'Epithet for Tarzan', 'Causes to grow, humorously', 'Car-washing equipment', \"'Alas!''\", 'Tab, essentially', 'Play again', '___ Championship', 'Founder of the Shondaland production company', 'Itchy layers', 'Locale for Ping-Pong, foosball, dancing, etc.', 'Safari destinations', 'U.N. secretary general Guterres', 'Quick', 'Most of the English force at Agincourt', \"'Goddess of the loud hunt,' in Homer's 'Iliad'\", 'Pulitzer category', 'What might be heard before a bust', 'Stab', 'Northern lapwing', 'Toys with strings', 'Did some shallow breathing?', \"Bootlicker's specialty\", 'Word with data or deal', 'M.M.A. decision', 'Swedish actress Lena', 'Make advances', 'Tense', 'Words to end a play', 'Break down, in a way', 'Chance to start fresh', '19th-century garb completed by top hats and gloves', \"'Let's have our cake and eat it, too!'\", 'Pillar of the superhero community', 'Heavenly being', \"Maguire of 2009's 'Brothers'\", 'Part of U.C.S.F.: Abbr.', 'Idaho senator Mike', \"It's a good thing\", 'Spot check?', 'Break down, in a way', 'Fictional Dr. Jones, familiarly', 'Big deal', 'Many a North African', 'Use a hitch on', 'People with a language of the same name']\n",
    "\n",
    "len(all_clues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5e28843",
   "metadata": {},
   "outputs": [],
   "source": [
    "json_puzzle_path = r\"./demo crosswords/crossword_1.json\"\n",
    "\n",
    "puzzle = json_CA_json_converter(json_puzzle_path, True)\n",
    "\n",
    "for dim in ['across', 'down']:\n",
    "    for grid_num in puzzle['clues'][dim].keys():\n",
    "        clue_answer_list = puzzle['clues'][dim][grid_num]\n",
    "        clue_section = clue_answer_list[0]\n",
    "        ans_section = clue_answer_list[1]\n",
    "        clue_section = clue_section.replace(\"&quot;\", \"\").replace(\"&#39;\", \"\")\n",
    "        puzzle['clues'][dim][grid_num] = [clue_section, ans_section]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b33c4597",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0db8e918",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24869e30",
   "metadata": {},
   "outputs": [],
   "source": [
    "crossword.letter_grid"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}