data_extract.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import string\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Coleção usada: http://disi.unitn.it/moschitti/corpora.htm\n",
    "\n",
    "Reuters-21578 -  90 categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleantext(text):\n",
    "    \"\"\"\n",
    "    Tranforma o texto para minúsculas\n",
    "    retira espaços e caracteres especiais \n",
    "    \"\"\"\n",
    "    text = ', '.join(text)\n",
    "    text = re.sub(r'\\d+', '', text)\n",
    "    text = text.replace('\\n',' ').lower().strip()\n",
    "    text = text.translate(str.maketrans(' ',' ', string.punctuation))\n",
    "    text = \" \".join(text.split())\n",
    "   \n",
    "    \n",
    "    return text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_extract(file):\n",
    "\n",
    "    \"\"\"\n",
    "    Extrai o texto das pastas.\n",
    "    Chama a função cleantext para limpar o texto\n",
    "    Salva o arquivo como arquivo csv\n",
    "    \"\"\"\n",
    "    \n",
    "    list_training = []\n",
    "    list_test = []\n",
    "    \n",
    "    for name_file_one in os.listdir(file):\n",
    "        path_one = os.path.realpath(file)+'/'+name_file_one\n",
    "        #print('name_file_one = ' + name_file_one)\n",
    "\n",
    "        for name_file_two in os.listdir(path_one):\n",
    "            path_two = os.path.realpath(path_one)+'/'+name_file_two\n",
    "            #print('name_file_two = ' + name_file_two)\n",
    "\n",
    "            for name_file_three in os.listdir(path_two):\n",
    "                path_three = os.path.realpath(path_two)+'/'+name_file_three\n",
    "                #print('name_file_three = ' + name_file_three)\n",
    "\n",
    "                with open(path_three, \"r\", errors=\"ignore\")  as archive:\n",
    "                    lines = archive.readlines()\n",
    "\n",
    "                data_tupla = (name_file_one, name_file_two, name_file_three, lines)   \n",
    "\n",
    "                if name_file_one == 'training':\n",
    "                    list_training.append(data_tupla)\n",
    "                elif name_file_one == 'test':\n",
    "                    list_test.append(data_tupla)\n",
    "                \n",
    "    dt_training = pd.DataFrame(list_training)\n",
    "    dt_training.columns=(['origem','topico', 'arquivo', 'texto'])\n",
    "    dt_training['texto'] = dt_training['texto'].apply(cleantext)\n",
    "    dt_training.to_csv('data_training.csv', index=False)\n",
    "\n",
    "    dt_test = pd.DataFrame(list_test)\n",
    "    dt_test.columns=(['origem', 'topico', 'arquivo', 'texto'])\n",
    "    dt_test['texto'] = dt_test['texto'].apply(cleantext)\n",
    "    dt_test.to_csv('data_test.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = 'Reuters21578-Apte-90Cat'\n",
    "#file = 'menorReuters21578-Apte-90Cat'\n",
    "\n",
    "data_extract(file)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "6e5b1ed3bef81dceac836bf818df7b07f0adde6e36c46e9fb6b1468e5bf6700d"
  },
  "kernelspec": {
   "display_name": "Python 3.8.12 64-bit ('ri_um': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}