preprocess-wiki.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c1d31f54",
   "metadata": {},
   "source": [
    "# Notebook for preprocessing Malay dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21eb8ed4",
   "metadata": {},
   "source": [
    "### Initilizing phonemizer and tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d6ca5ee4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "\n",
    "config_path = \"Configs/config.yml\"\n",
    "config = yaml.safe_load(open(config_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b52b79ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`openai-whisper` is not available, native whisper processor is not available, will use huggingface processor instead.\n"
     ]
    }
   ],
   "source": [
    "from phonemize import phonemize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2b363b0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import phonemizer\n",
    "global_phonemizer = phonemizer.backend.EspeakBackend(language='ms', preserve_punctuation=True,  with_stress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "003bbb16",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(config['dataset_params']['tokenizer'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2eb25417",
   "metadata": {},
   "source": [
    "### Process dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "25e5ae16",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "dataset = load_dataset(\"wikimedia/wikipedia\", \"20231101.ms\")['train']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5f27e160",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!rm -rf wiki_phoneme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ca7ca2f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "root_directory = \"./wiki_phoneme\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "92a578d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "num_shards = 1000\n",
    "\n",
    "def process_shard(i):\n",
    "    directory = root_directory + \"/shard_\" + str(i)\n",
    "    if os.path.exists(directory):\n",
    "        print(\"Shard %d already exists!\" % i)\n",
    "        return\n",
    "    print('Processing shard %d ...' % i)\n",
    "    shard = dataset.shard(num_shards=num_shards, index=i)\n",
    "    processed_dataset = shard.map(lambda t: phonemize(t['text'], global_phonemizer, tokenizer), remove_columns=['text'])\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "    processed_dataset.save_to_disk(directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d73caf6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pebble import ProcessPool\n",
    "from concurrent.futures import TimeoutError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "04261364",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "max_workers = 20\n",
    "\n",
    "with ProcessPool(max_workers=max_workers) as pool:\n",
    "    pool.map(process_shard, range(num_shards))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b78caee6",
   "metadata": {},
   "source": [
    "### Collect all shards to form the processed dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0568da38",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_from_disk, concatenate_datasets\n",
    "\n",
    "output = [dI for dI in os.listdir(root_directory) if os.path.isdir(os.path.join(root_directory,dI))]\n",
    "datasets = []\n",
    "for o in output:\n",
    "    directory = root_directory + \"/\" + o\n",
    "    try:\n",
    "        shard = load_from_disk(directory)\n",
    "        datasets.append(shard)\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c1547f2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d644581eaf94001979903cf5dfa40ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/3 shards):   0%|          | 0/366783 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset saved to wikipedia_20220301.en.processed\n"
     ]
    }
   ],
   "source": [
    "dataset = concatenate_datasets(datasets)\n",
    "dataset.save_to_disk(config['data_folder'])\n",
    "print('Dataset saved to %s' % config['data_folder'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7b6d70ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.1G\twikipedia_20220301.en.processed\r\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!du -hs wikipedia_20220301.en.processed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "39e0643f",
   "metadata": {},
   "outputs": [],
   "source": [
    "news_dataset = load_from_disk('news')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "fdcef824",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['id', 'url', 'title', 'input_ids', 'phonemes'],\n",
       "    num_rows: 1571960\n",
       "})"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = concatenate_datasets([dataset, news_dataset])\n",
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cdf6f6f6",
   "metadata": {},
   "source": [
    "### Remove unneccessary tokens from the pre-trained tokenizer\n",
    "The pre-trained tokenizer contains a lot of tokens that are not used in our dataset, so we need to remove these tokens. We also want to predict the word in lower cases because cases do not matter that much for TTS. Pruning the tokenizer is much faster than training a new tokenizer from scratch. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "28cec407",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from simple_loader import FilePathDataset, build_dataloader\n",
    "\n",
    "file_data = FilePathDataset(dataset)\n",
    "loader = build_dataloader(file_data, num_workers=10, batch_size=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "0b7504eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "special_token = config['dataset_params']['word_separator']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "14dc28f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "token_maps = {}\n",
    "for k, v in tokenizer.vocab.items():\n",
    "    token_maps[v] = {'word': k, 'token': v}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "c1c94be2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token mapper saved to token_maps.pkl\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "with open(config['dataset_params']['token_maps'], 'wb') as handle:\n",
    "    pickle.dump(token_maps, handle)\n",
    "print('Token mapper saved to %s' % config['dataset_params']['token_maps'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "51aaeeca",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('token_maps.pkl', 'rb') as fopen:\n",
    "    token_maps = pickle.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "44a56c92",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "300000"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(token_maps)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c9e968e",
   "metadata": {},
   "source": [
    "### Test the dataset with dataloader\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "f9025e2f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "177\n"
     ]
    }
   ],
   "source": [
    "from dataloader import build_dataloader\n",
    "\n",
    "train_loader = build_dataloader(dataset, batch_size=10, num_workers=0, dataset_config=config['dataset_params'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "e01f7a14",
   "metadata": {},
   "outputs": [],
   "source": [
    "phonemes = dataset[1]['phonemes']\n",
    "input_ids = dataset[1]['input_ids']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "70874215",
   "metadata": {},
   "outputs": [],
   "source": [
    "_, (words, labels, phonemes, input_lengths, masked_indices) = next(enumerate(train_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "3cacd27b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10, 238])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "6f5f7d90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 3778,  3778,  3778,  3778,  3778,  3778,     2,   479,   479,   479,\n",
       "          479,   479,   479,   479,   479,     2,    31,    31,    31,    31,\n",
       "           31,    31,    31,    31,    31,    31,     2,    32,    32,    32,\n",
       "           32,    32,    32,    32,     2, 10606, 10606, 10606, 10606, 10606,\n",
       "        10606, 10606,     2,    11,    11,    11,     2,   151,   151,   151,\n",
       "          151,   151,   151,   151,   151,   151,   151,     2,    45,    45,\n",
       "            2,    18,    18,    18,    18,    18,    18,     2,    73,    73,\n",
       "           73,    73,    73,    73,    73,    73,     2,   169,   169,   169,\n",
       "          169,   169,   169,   169,   169,   169,   169,   169,     2,  6361,\n",
       "         6361,  6361,  6361,  6361,  6361,     2,     5,     2,   245,   245,\n",
       "          245,   245,   245,   245,   245,   245,   245,   245,   245,     2,\n",
       "         3185,  3185,  3185,  3185,  3185,  3185,  3185,  3185,     2,   113,\n",
       "          113,   113,   113,   113,   113,   113,   113,     2,     5,     2,\n",
       "          310,   310,   310,   310,   310,   310,   310,   310,   310,     2,\n",
       "         1366,  1366,  1366,  1366,  1366,  1366,  1366,  1366,  1366,  1366,\n",
       "            2,   114,   114,   114,   114,   114,   114,     2,     5,     2,\n",
       "           78,    78,    78,    78,    78,    78,    78,    78,    78,    78,\n",
       "           78,    78,     2,     6,     2,    57,    57,    57,    57,    57,\n",
       "           57,    57,     2,    48,    48,    48,    48,    48,     2,  4302,\n",
       "         4302,  4302,  4302,  4302,  4302,     2,   440,   440,   440,   440,\n",
       "            2,  5101,  5101,  5101,  5101,  5101,  5101,     2,   245,   245,\n",
       "          245,   245,   245,   245,   245,   245,   245,   245,   245,     2,\n",
       "         3185,  3185,  3185,  3185,  3185,  3185,  3185,  3185,     2,   113,\n",
       "          113,   113,   113,   113,   113,   113,   113,     2])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "d5ea0331",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'gurun gurun gurun gurun gurun gurun [SEP] panjang panjang panjang panjang panjang panjang panjang panjang [SEP] merupakan merupakan merupakan merupakan merupakan merupakan merupakan merupakan merupakan merupakan [SEP] sebuah sebuah sebuah sebuah sebuah sebuah sebuah [SEP] nagari nagari nagari nagari nagari nagari nagari [SEP] yang yang yang [SEP] termasuk termasuk termasuk termasuk termasuk termasuk termasuk termasuk termasuk termasuk [SEP] ke ke [SEP] dalam dalam dalam dalam dalam dalam [SEP] wilayah wilayah wilayah wilayah wilayah wilayah wilayah wilayah [SEP] kecamatan kecamatan kecamatan kecamatan kecamatan kecamatan kecamatan kecamatan kecamatan kecamatan kecamatan [SEP] bayang bayang bayang bayang bayang bayang [SEP] , [SEP] kabupaten kabupaten kabupaten kabupaten kabupaten kabupaten kabupaten kabupaten kabupaten kabupaten kabupaten [SEP] pesisir pesisir pesisir pesisir pesisir pesisir pesisir pesisir [SEP] selatan selatan selatan selatan selatan selatan selatan selatan [SEP] , [SEP] provinsi provinsi provinsi provinsi provinsi provinsi provinsi provinsi provinsi [SEP] sumatera sumatera sumatera sumatera sumatera sumatera sumatera sumatera sumatera sumatera [SEP] barat barat barat barat barat barat [SEP] , [SEP] indonesia indonesia indonesia indonesia indonesia indonesia indonesia indonesia indonesia indonesia indonesia indonesia [SEP] . [SEP] pautan pautan pautan pautan pautan pautan pautan [SEP] luar luar luar luar luar [SEP] situs situs situs situs situs situs [SEP] web web web web [SEP] resmi resmi resmi resmi resmi resmi [SEP] kabupaten kabupaten kabupaten kabupaten kabupaten kabupaten kabupaten kabupaten kabupaten kabupaten kabupaten [SEP] pesisir pesisir pesisir pesisir pesisir pesisir pesisir pesisir [SEP] selatan selatan selatan selatan selatan selatan selatan selatan [SEP]'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(words[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "ef09b1c6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 92, 156,  63,  ...,  43,  56,  16],\n",
       "        [ 61,  83,  55,  ...,  16,   0,   0],\n",
       "        [ 61,  83,  55,  ...,   0,   0,   0],\n",
       "        ...,\n",
       "        [ 53,  83,  46,  ...,   0,   0,   0],\n",
       "        [ 92,  43,  44,  ...,   0,   0,   0],\n",
       "        [ 56,  83, 156,  ...,   0,   0,   0]])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "phonemes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "21d0f682",
   "metadata": {},
   "outputs": [],
   "source": [
    "from text_utils import dicts\n",
    "\n",
    "rev_dicts = {v: k for k, v in dicts.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "bf7a78b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'səməntˈarə ˈitu , bəlˈiaʊ MMM MMMMMM ˈahli parlˈimən MMMMMMM pˈantaɪ tˈurot mənˌasihˈatkan kaˈaə MMMMMM MMMMMMMMM , stʃˈam , ˈuntoʔ məlapˈɔrkan kədʒadˈian jaŋ mənˈimpə mərˈɛkə dan tˈidaʔ mˈalu ˈuntoʔ bərhˈuboŋ miˈaaə pˈihaʔ bərkuˈasə . $$'"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join([rev_dicts[int(i)] for i in phonemes[1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "a686e331",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a4456e764f540789aea3abf95ee331b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/4 shards):   0%|          | 0/1571960 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset.save_to_disk(config['data_folder'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "731615f4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d60e15ea745406cb785e478171aef8d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a9d6f24a7e94a0da00dfe27b79c824e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/393 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65990cdf152c4851bc12cfa39c4aa5d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/393 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5700c849359246c0a33b72f1a0355648",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/393 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0a0303fa03f4fc291e63d826f9a7455",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/393 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/datasets/mesolitica/PL-BERT-MS/commit/41caa94b49727e8f10248a683508cc8d4b8ef34f', commit_message='Upload dataset', commit_description='', oid='41caa94b49727e8f10248a683508cc8d4b8ef34f', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/mesolitica/PL-BERT-MS', endpoint='https://huggingface.co', repo_type='dataset', repo_id='mesolitica/PL-BERT-MS'), pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.push_to_hub('mesolitica/PL-BERT-MS')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b4ccd119",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1015c9eaa51644e4945dff4bfe14f580",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/802 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d575cbc0f1a44718b862bced22e3ce0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00000-of-00004.parquet:   0%|          | 0.00/322M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "098c7d63975a42ff84ff5337534d5e39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00001-of-00004.parquet:   0%|          | 0.00/39.5M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e27f4d8eb3d7430298e4b42d76e24eec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00002-of-00004.parquet:   0%|          | 0.00/54.9M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "097a481c856f4589af4dbfcc55203e33",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00003-of-00004.parquet:   0%|          | 0.00/57.1M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a589e1e0f7cd4ec18c555a7967a8fefb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/1571960 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "dataset = load_dataset(\"mesolitica/PL-BERT-MS\", split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8479b283",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['id', 'url', 'title', 'input_ids', 'phonemes'],\n",
       "    num_rows: 1571960\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "61468447",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['id', 'url', 'title', 'input_ids', 'phonemes'],\n",
       "    num_rows: 1571960\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_from_disk\n",
    "\n",
    "dataset2 = load_from_disk('wikipedia_20220301.en.processed')\n",
    "dataset2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4f1cea7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '17218',\n",
       " 'url': 'https://ms.wikipedia.org/wiki/Pelabuhan%20Kerteh',\n",
       " 'title': 'Pelabuhan Kerteh',\n",
       " 'input_ids': [1244,\n",
       "  10745,\n",
       "  441,\n",
       "  7,\n",
       "  10745,\n",
       "  5,\n",
       "  367,\n",
       "  76,\n",
       "  2507,\n",
       "  1275,\n",
       "  3036,\n",
       "  4475,\n",
       "  9,\n",
       "  1154,\n",
       "  647,\n",
       "  11,\n",
       "  10713,\n",
       "  7,\n",
       "  48,\n",
       "  530,\n",
       "  367,\n",
       "  9,\n",
       "  11,\n",
       "  150356,\n",
       "  8040,\n",
       "  7,\n",
       "  40,\n",
       "  3841,\n",
       "  10745,\n",
       "  6,\n",
       "  1244,\n",
       "  7,\n",
       "  43],\n",
       " 'phonemes': ['pəlabˈuhan',\n",
       "  'kˈɛɾrteh',\n",
       "  'dibˈinə',\n",
       "  'di',\n",
       "  'kˈɛɾrteh',\n",
       "  ',',\n",
       "  'trəŋɡˈanu',\n",
       "  'bˈaɡi',\n",
       "  'mənˈampoŋ',\n",
       "  'kəpəɾrlˈuan',\n",
       "  'pəŋhantˈaran',\n",
       "  'pətrolˈɛom',\n",
       "  'dan',\n",
       "  'ɡˈas',\n",
       "  'ˈasli',\n",
       "  'jaŋ',\n",
       "  'diɡˈali',\n",
       "  'di',\n",
       "  'lˈuar',\n",
       "  'pˈantaɪ',\n",
       "  'trəŋɡˈanu',\n",
       "  'dan',\n",
       "  'jaŋ',\n",
       "  'ditˈɛlah',\n",
       "  'diprˈoses',\n",
       "  'di',\n",
       "  'kawˈasan',\n",
       "  'darˈatan',\n",
       "  'kˈɛɾrteh',\n",
       "  '.',\n",
       "  'pəlabˈuhan',\n",
       "  'di',\n",
       "  'məlˈesiə']}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a5d670c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '17218',\n",
       " 'url': 'https://ms.wikipedia.org/wiki/Pelabuhan%20Kerteh',\n",
       " 'title': 'Pelabuhan Kerteh',\n",
       " 'input_ids': [1244,\n",
       "  10745,\n",
       "  441,\n",
       "  7,\n",
       "  10745,\n",
       "  5,\n",
       "  367,\n",
       "  76,\n",
       "  2507,\n",
       "  1275,\n",
       "  3036,\n",
       "  4475,\n",
       "  9,\n",
       "  1154,\n",
       "  647,\n",
       "  11,\n",
       "  10713,\n",
       "  7,\n",
       "  48,\n",
       "  530,\n",
       "  367,\n",
       "  9,\n",
       "  11,\n",
       "  150356,\n",
       "  8040,\n",
       "  7,\n",
       "  40,\n",
       "  3841,\n",
       "  10745,\n",
       "  6,\n",
       "  1244,\n",
       "  7,\n",
       "  43],\n",
       " 'phonemes': ['pəlabˈuhan',\n",
       "  'kˈɛɾrteh',\n",
       "  'dibˈinə',\n",
       "  'di',\n",
       "  'kˈɛɾrteh',\n",
       "  ',',\n",
       "  'trəŋɡˈanu',\n",
       "  'bˈaɡi',\n",
       "  'mənˈampoŋ',\n",
       "  'kəpəɾrlˈuan',\n",
       "  'pəŋhantˈaran',\n",
       "  'pətrolˈɛom',\n",
       "  'dan',\n",
       "  'ɡˈas',\n",
       "  'ˈasli',\n",
       "  'jaŋ',\n",
       "  'diɡˈali',\n",
       "  'di',\n",
       "  'lˈuar',\n",
       "  'pˈantaɪ',\n",
       "  'trəŋɡˈanu',\n",
       "  'dan',\n",
       "  'jaŋ',\n",
       "  'ditˈɛlah',\n",
       "  'diprˈoses',\n",
       "  'di',\n",
       "  'kawˈasan',\n",
       "  'darˈatan',\n",
       "  'kˈɛɾrteh',\n",
       "  '.',\n",
       "  'pəlabˈuhan',\n",
       "  'di',\n",
       "  'məlˈesiə']}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset2[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4bded33e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3.10",
   "language": "python",
   "name": "python3.10"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}