UNIPASSU_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import argparse\n",
    "import time\n",
    "import os\n",
    "import sys\n",
    "import random\n",
    "import numpy as np\n",
    "import math\n",
    "import os, json, copy\n",
    "import pandas as pd\n",
    "import torch\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "from torch.utils.data import DataLoader,TensorDataset\n",
    "from torch.utils.data import Subset\n",
    "import os, json, copy\n",
    "import numpy as np\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from pandas.core.frame import DataFrame\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import random\n",
    "import time, math\n",
    "import logging\n",
    "import torch.nn as nn\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "import torch.nn.functional as f\n",
    "import torch.nn.functional as F\n",
    "\n",
    "configs={\n",
    "  \"mode\": \"train_mixed\",\n",
    "  \"seed\": 11,\n",
    "  \"batch_size\": 256,\n",
    "  \"device\": \"cuda\",\n",
    "  \"colab\": False,\n",
    "  \"num_workers\": 3,\n",
    "  \"lr\": 0.001,\n",
    "  \"weight_decay\": 0.0005,\n",
    "  \"epochs\": 50,\n",
    "  \"lr_decay\": 5,\n",
    "  \"lr_decay_rate\": 0.8,\n",
    "  \"custom_loss\": \"fbeta_loss\",\n",
    "  \"preprocess\": True,\n",
    "  \"split_dataset\": False,\n",
    "  \"lambda\": 0.3,\n",
    "  \"beta\": 15.0,\n",
    "  \"split_ratio\":0.8,\n",
    "  \"only_train\":True,\n",
    "}\n",
    "use_cuda = torch.cuda.is_available()\n",
    "device = torch.device(\"cuda\" if use_cuda and configs['device'] =='cuda' else \"cpu\")\n",
    "configs['device']=str(device)\n",
    "time_data = time.strftime('%m-%d_%H-%M-%S', time.localtime(time.time()))\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from pandas.core.frame import DataFrame\n",
    "\n",
    "def fix_seed(seed):\n",
    "    #Seed 고정\n",
    "    random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.random.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Fix Seed ###\n",
    "\n",
    "random.seed(configs['seed'])\n",
    "torch.manual_seed(configs['seed'])\n",
    "torch.random.manual_seed(configs['seed'])\n",
    "torch.cuda.manual_seed_all(configs['seed'])\n",
    "np.random.seed(configs['seed'])\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "### data path designation ###\n",
    "current_path=os.getcwd()\n",
    "if configs['colab']==True:\n",
    "    par_path=os.path.abspath(os.path.join(current_path, os.pardir))\n",
    "    current_path=os.path.join(par_path,'drive','MyDrive')\n",
    "data_path=os.path.join(current_path)\n",
    "save_path=os.path.join(current_path,'training_data')\n",
    "if os.path.exists(save_path) == False:\n",
    "    os.mkdir(save_path)\n",
    "if os.path.exists(os.path.join(save_path,time_data)) == False:\n",
    "    os.mkdir(os.path.join(save_path,time_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2021-07-21 12:18:07,566] [main] C:\\anaconda3\\envs\\torch\\lib\\site-packages\\ipykernel_launcher.py -f C:\\Users\\Medical IT\\AppData\\Roaming\\jupyter\\runtime\\kernel-f23957ab-f8d7-4ce5-bd75-446f07f26e54.json\n"
     ]
    }
   ],
   "source": [
    "## save configuration ##\n",
    "def load_params(configs,current_path, file_name):\n",
    "    ''' replay_name from file_name '''\n",
    "    with open(os.path.join(current_path, 'training_data', '{}.json'.format(file_name)), 'r') as fp:\n",
    "        configs = json.load(fp)\n",
    "    return configs\n",
    "\n",
    "\n",
    "def save_params(configs,current_path, time_data):\n",
    "    with open(os.path.join(current_path, 'training_data', '{}.json'.format(time_data)), 'w') as fp:\n",
    "        json.dump(configs, fp, indent=2)\n",
    "\n",
    "save_params(configs,current_path,time_data)\n",
    "########################\n",
    "\n",
    "## logger ##\n",
    "def set_logging_defaults(time_data,logdir):\n",
    "    # set basic configuration for logging\n",
    "    logging.basicConfig(format=\"[%(asctime)s] [%(name)s] %(message)s\",\n",
    "                        level=logging.INFO,\n",
    "                        handlers=[logging.FileHandler(os.path.join(logdir,time_data, 'log.txt')),\n",
    "                                  logging.StreamHandler(os.sys.stdout)])\n",
    "\n",
    "    # log cmdline argumetns\n",
    "    logger = logging.getLogger('main')\n",
    "    logger.info(' '.join(os.sys.argv))\n",
    "\n",
    "set_logging_defaults(time_data,save_path)\n",
    "logger = logging.getLogger('main')\n",
    "############\n",
    "\n",
    "## calculation of the score ##\n",
    "def calc_score(predictions, targets,score_dict):\n",
    "    predictions=predictions.detach().clone().cpu()\n",
    "    targets=targets.detach().clone().cpu()\n",
    "    accuracy = accuracy_score(targets, predictions,)\n",
    "    if targets.unique().size(0)==3:\n",
    "        avg='macro'\n",
    "    elif targets.unique().size(0)==2:\n",
    "        avg='binary'\n",
    "    precision = precision_score(targets, predictions,average=avg,zero_division=0)\n",
    "    recall = recall_score(targets, predictions,average=avg,zero_division=0)\n",
    "    f1 = f1_score(targets, predictions,average=avg,zero_division=0)\n",
    "    score_dict['total']+=targets.size(0)\n",
    "    score_dict['accuracy'] = accuracy*100.0\n",
    "    score_dict['precision'] = precision*100.0\n",
    "    score_dict['recall'] = recall*100.0\n",
    "    score_dict['f1score']=f1*100.0\n",
    "    return score_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train.csv의 column을 학습에서 feature로 사용허가 위해서 preprocessing을 진행 할 때에, 각 고유분류코드의 우범 및 핵심우범을 저지를 기댓값에 따라 가중치를 주기 위해\n",
    "# 기댓값을 계산하여 Json file로 저장하는 Class입니다.\n",
    "# 추가적으로, train.csv의 컬럼 데이터 이외에 컬럼 데이터를 조합하거나 가공해서 만든 새로운 데이터 컬럼에 대해서도 계산을 진행합니다.\n",
    "\n",
    "class MappingJsonGenerator():\n",
    "    def __init__(self, train_csv, fillna_str, drop_list, only_train, data_path = None):\n",
    "        \n",
    "        self.only_train = only_train\n",
    "        self.train_csv=train_csv\n",
    "        self.fillna_str = fillna_str\n",
    "        self.drop_list = drop_list\n",
    "        \n",
    "        self.train_csv['HS_upper'] = self.train_csv['HS10단위부호'] // 100000000\n",
    "        self.train_csv['HS_middle'] = self.train_csv['HS10단위부호'] // 1000000\n",
    "        self.train_csv['HS_low'] = self.train_csv['HS10단위부호'] // 10000\n",
    "\n",
    "        self.train_csv['관세율구분코드_1자리']=self.train_csv['관세율구분코드'].str.slice(start = 0, stop = 1)\n",
    "\n",
    "        self.train_csv['단위무게(KG)가격'] = (self.train_csv['과세가격원화금액']/self.train_csv['신고중량(KG)']).map(lambda x: np.round(x, 0)).map(str)\n",
    "\n",
    "        self.train_csv.drop(['과세가격원화금액', '신고중량(KG)'], axis = 1,errors='ignore',inplace=True)\n",
    "        \n",
    "        if self.only_train:\n",
    "            # train_indices = np.load('./data/custom_contest/train_indices.npy')\n",
    "            train_indices = np.load(os.path.join(data_path,'train_indices.npy'))\n",
    "            self.train_valid_csv = self.train_csv\n",
    "            self.train_csv = self.train_valid_csv[self.train_valid_csv.columns].iloc[train_indices]\n",
    "            self.train_valid_csv=self.train_valid_csv.drop(['우범여부', '핵심적발'] + self.drop_list, axis = 1,errors='ignore')\n",
    "            self.train_valid_csv = self.train_valid_csv.fillna(self.fillna_str)\n",
    "\n",
    "\n",
    "        self.crime = np.array(self.train_csv['우범여부'])\n",
    "        self.priority = np.array(self.train_csv['핵심적발'])\n",
    "        self.train_csv=self.train_csv.drop(['우범여부', '핵심적발'] + self.drop_list, axis = 1,errors='ignore')\n",
    "\n",
    "        self.train_csv = self.train_csv.fillna(self.fillna_str)\n",
    "        self.column_list = np.array(self.train_csv.columns, dtype=str)\n",
    "\n",
    "        self.crime_idx = np.where(self.crime == 1)[0]\n",
    "        self.non_priority_idx = np.where(self.priority == 1)[0]\n",
    "        self.priority_idx = np.where(self.priority == 2)[0]\n",
    "        self.dictionary = dict()\n",
    "\n",
    "        self.crime_threshold_no_valid = [0]*21\n",
    "        self.priority_threshold_no_valid = [5]*21\n",
    "        \n",
    "    def __call__(self):\n",
    "        return self.forward()\n",
    "    \n",
    "    def forward(self):\n",
    "        train_np = np.array(self.train_csv, dtype = str)\n",
    "        print(train_np.shape)\n",
    "        if self.only_train:\n",
    "            train_valid_np = np.array(self.train_valid_csv, dtype = str)\n",
    "            print('calc count_ratio for only train')\n",
    "            print(train_valid_np.shape)\n",
    "        else:\n",
    "            train_valid_np = train_np\n",
    "            print('calc count_ratio for train + valid')\n",
    "            print(train_np.shape)\n",
    "\n",
    "        for i, col in enumerate(self.column_list):\n",
    "            crime_count_mean, crime_ratio_mean = 0., 0.\n",
    "            priority_count_mean, priority_ratio_mean = 0., 0.\n",
    "            self.dictionary[col] = {}\n",
    "            concat = np.concatenate([train_valid_np[:, i]], axis = 0)\n",
    "            total_code, total_count = np.unique(concat, return_counts=True)\n",
    "            crime_code, crime_count = np.unique(train_np[:, i][self.crime_idx], return_counts=True)\n",
    "\n",
    "            _, total_priority_count = np.unique(np.concatenate((train_np[:, i][self.priority_idx],train_np[:, i][self.non_priority_idx]),axis=0), return_counts=True)\n",
    "            priority_code, priority_count = np.unique(train_np[:, i][self.priority_idx], return_counts=True)\n",
    "\n",
    "            crime_ratio = np.empty((total_count.shape[0], ))\n",
    "            \n",
    "            priority_ratio = np.empty((total_count.shape[0], ))\n",
    "\n",
    "            c_idx = 0\n",
    "            p_idx = 0\n",
    "            for assign_idx, c in enumerate(total_code) : \n",
    "                self.dictionary[col][c] = {}\n",
    "                self.dictionary[col][c]['total_count'] = int(total_count[assign_idx])\n",
    "\n",
    "                if c not in crime_code : \n",
    "                    crime_ratio[assign_idx] = 0.\n",
    "                    self.dictionary[col][c]['crime_count'] = int(0)\n",
    "                    self.dictionary[col][c]['is_crime_mask'] = True\n",
    "                else :\n",
    "                    crime_ratio[assign_idx] = np.round(crime_count[c_idx] / total_count[assign_idx], 4)\n",
    "                    self.dictionary[col][c]['crime_count'] = int(crime_count[c_idx])\n",
    "                    crime_count_mean += float(crime_count[c_idx])\n",
    "                    crime_ratio_mean += float(crime_ratio[assign_idx]*crime_count[c_idx])\n",
    "                    if crime_count[c_idx] <= self.crime_threshold_no_valid[i]:\n",
    "                        self.dictionary[col][c]['is_crime_mask'] = True\n",
    "                    else : \n",
    "                        self.dictionary[col][c]['is_crime_mask'] = False\n",
    "                    c_idx += 1\n",
    "\n",
    "                self.dictionary[col][c]['crime_ratio'] = float(crime_ratio[assign_idx])\n",
    "                self.dictionary[col][c]['onehot'] = int(assign_idx+1)\n",
    "            \n",
    "            crime_concat = np.concatenate([crime_ratio.reshape(-1, 1), total_code.reshape(-1, 1)], axis = 1)\n",
    "            crime_concat = np.array(sorted(crime_concat, key = lambda x : x[0], reverse=True))\n",
    "\n",
    "            for k, c in enumerate(crime_concat[:, 1]):\n",
    "                self.dictionary[col][c]['sorted_crime_onehot'] = int(k+1)\n",
    "\n",
    "\n",
    "                \n",
    "            for assign_idx,p in enumerate(total_code):\n",
    "                if p not in priority_code: \n",
    "                    priority_ratio[assign_idx] = 0.\n",
    "                    self.dictionary[col][p]['priority_count'] = int(0)\n",
    "                    self.dictionary[col][p]['is_priority_mask'] = True\n",
    "                else :\n",
    "                    assign_priority_idx=np.where(priority_code==p)[0]\n",
    "                    #priority_ratio[assign_idx] = np.round(priority_count[p_idx] / total_priority_count[assign_priority_idx], 4)\n",
    "                    priority_ratio[assign_idx] = np.round(priority_count[p_idx] / self.dictionary[col][p]['crime_count'], 4)\n",
    "                    self.dictionary[col][p]['priority_count'] = int(priority_count[p_idx])\n",
    "                    priority_count_mean += float(priority_count[p_idx])\n",
    "                    priority_ratio_mean += float(priority_ratio[assign_idx]*priority_count[p_idx])\n",
    "                    if priority_count[p_idx] <= self.priority_threshold_no_valid[i]:\n",
    "                        self.dictionary[col][p]['is_priority_mask'] = True\n",
    "                    else : \n",
    "                        self.dictionary[col][p]['is_priority_mask'] = False\n",
    "                    p_idx += 1\n",
    "\n",
    "                \n",
    "                self.dictionary[col][p]['priority_ratio'] = float(priority_ratio[assign_idx])\n",
    "\n",
    "            priority_concat = np.concatenate([priority_ratio.reshape(-1, 1), total_code.reshape(-1, 1)], axis = 1)\n",
    "            priority_concat = np.array(sorted(priority_concat, key = lambda x : x[0], reverse=True))\n",
    "\n",
    "            for k, c in enumerate(priority_concat[:, 1]):\n",
    "                self.dictionary[col][c]['sorted_priority_onehot'] = int(k+1)\n",
    "\n",
    "            self.dictionary[col]['crime_count_mean'] = crime_count_mean / train_valid_np.shape[0]\n",
    "            self.dictionary[col]['crime_ratio_mean'] =crime_ratio_mean / train_valid_np.shape[0]\n",
    "            self.dictionary[col]['priority_count_mean'] = priority_count_mean / train_valid_np.shape[0]\n",
    "            self.dictionary[col]['priority_ratio_mean'] = priority_ratio_mean / train_valid_np.shape[0]\n",
    "\n",
    "        return self.dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 위에서 설명드린 Class를 이용하여 Json file을 생성하고 해당 정보를 이용하여 preprocessing을 진행하는 Class입니다.\n",
    "# 우범여부를 학습하기 위한 데이터에 사용되는 _crime_transform과 핵심적발을 학습하기 위한 데이터에 사용되는 _priority_transform이 주요 동작을 수행합니다.\n",
    "# 추가적으로, train.csv의 컬럼 데이터 이외에 컬럼 데이터를 조합하거나 가공해서 만든 새로운 데이터 컬럼에 대해서도 계산을 진행합니다.\n",
    "\n",
    "def find_digits(x):\n",
    "    temp = math.log2(x)\n",
    "    if temp == math.floor(temp):\n",
    "        n = temp\n",
    "    else:\n",
    "        n = math.floor(temp) + 1\n",
    "    return int(n)\n",
    "\n",
    "\n",
    "def binary_transform(x):\n",
    "    if x == 0:\n",
    "        return '0'\n",
    "    else:\n",
    "        binary=''\n",
    "        while x>0:\n",
    "            x, mod = divmod(x,2)\n",
    "            binary += str(mod)\n",
    "        return binary\n",
    "\n",
    "\n",
    "class RescaleNumeric:\n",
    "    def __init__(self):\n",
    "        self.minmax_scale=preprocessing.MinMaxScaler(feature_range=(0,1))\n",
    "    def __call__(self,x):\n",
    "        x_scale=self.minmax_scale.fit_transform(np.log(x+1).reshape(-1,1))\n",
    "        return x_scale\n",
    "\n",
    "\n",
    "class Preprocessing:\n",
    "    def __init__(self,data_path,configs):\n",
    "        self.configs=configs\n",
    "        self.data_path=data_path\n",
    "        # load mapping dictionary\n",
    "        train_dataframe,test_dataframe=self._load_dataset()\n",
    "        npy_dict=dict()\n",
    "        npy_dict['crime_targets']=train_dataframe.pop('우범여부')\n",
    "        npy_dict['priority_targets']=train_dataframe.pop('핵심적발')\n",
    "        npy_dict['train_indices'], npy_dict['valid_indices']=self._split_indices(train_dataframe,npy_dict['priority_targets'])\n",
    "        for key in npy_dict.keys():\n",
    "            if isinstance(npy_dict,DataFrame):\n",
    "                npy_dict[key]=npy_dict[key].to_numpy()\n",
    "            np.save(os.path.join(self.data_path,'{}.npy'.format(key)),npy_dict[key])\n",
    "        del npy_dict\n",
    "        train_dataframe,test_dataframe=self._load_dataset()\n",
    "        self.mapping_dict = MappingJsonGenerator(train_dataframe,'Missing', ['신고번호', '신고일자', '관세율','검사결과코드'], self.configs['only_train'], data_path)() #'과세가격원화금액', '신고중량(KG)',\n",
    "        print(\"Generate Json complete\")\n",
    "        # save\n",
    "        with open(os.path.join(data_path,'mapping.json'), 'w') as fp:\n",
    "            json.dump(self.mapping_dict, fp, indent=2)\n",
    "        # load\n",
    "        with open(os.path.join(data_path,'mapping.json'), 'r') as fp:\n",
    "            self.mapping_dict = copy.deepcopy(json.load(fp))\n",
    "\n",
    "    def _load_dataset(self):\n",
    "        train_dataframe=copy.deepcopy(pd.read_csv(os.path.join(self.data_path,'train.csv')))\n",
    "        test_dataframe=copy.deepcopy(pd.read_csv(os.path.join(self.data_path,'test.csv')))\n",
    "        return train_dataframe,test_dataframe\n",
    "\n",
    "    def run(self) ->dict:\n",
    "        npy_dict={}\n",
    "        #load dataset\n",
    "        train_dataframe,test_dataframe=self._load_dataset()\n",
    "        if self.configs['mode']=='record':\n",
    "            data_frame_list=[test_dataframe]\n",
    "            data_type_list=['test']\n",
    "        else:\n",
    "            data_frame_list=[train_dataframe,test_dataframe]\n",
    "            data_type_list=['train','test']\n",
    "        #transform and save the dataset\n",
    "        for csv_dataframe,data_type in zip(data_frame_list,data_type_list):\n",
    "            npy_dict['{}_priority_data'.format(data_type)]=self._priority_transform(copy.deepcopy(csv_dataframe))\n",
    "            npy_dict['{}_crime_data'.format(data_type)]=self._crime_transform(copy.deepcopy(csv_dataframe))\n",
    "        for key in npy_dict.keys():\n",
    "            if isinstance(npy_dict,DataFrame):\n",
    "                npy_dict[key]=npy_dict[key].to_numpy()\n",
    "            np.save(os.path.join(self.data_path,'{}.npy'.format(key)),npy_dict[key])\n",
    "        return npy_dict\n",
    "\n",
    "    def _split_indices(self,dataframe:DataFrame,targets:np.ndarray)->np.ndarray:\n",
    "        indices=np.arange(len(dataframe))\n",
    "        train_indices,valid_indices=train_test_split(indices,stratify=targets,random_state=self.configs['seed'],test_size=1-self.configs['split_ratio'],train_size=self.configs['split_ratio'])\n",
    "        return train_indices, valid_indices\n",
    "\n",
    "    def _crime_transform(self, dataframe:DataFrame)->np.ndarray:\n",
    "        rescaler=RescaleNumeric()\n",
    "        \"\"\"\n",
    "        categorical_features = ['통관지세관부호', '신고인부호', '수입자부호', '해외거래처부호', '특송업체부호', \n",
    "                                '수입통관계획코드', '수입신고구분코드', '수입거래구분코드', '수입종류코드', \n",
    "                                '징수형태코드', '운송수단유형코드', '반입보세구역부호', 'HS_upper', 'HS_middle', \n",
    "                                '적출국가코드', '원산지국가코드', '관세율구분코드']\n",
    "        \"\"\"\n",
    "        categorical_features = self.mapping_dict.keys()\n",
    "        dataframe['HS_upper'] = dataframe['HS10단위부호'] // 100000000 # 위 2자리\n",
    "        dataframe['HS_middle'] = dataframe['HS10단위부호'] // 1000000 # 위 4자리\n",
    "        dataframe['HS_low'] = dataframe['HS10단위부호'] // 10000 # 위 4자리\n",
    "        dataframe['관세율구분코드_1자리']=dataframe['관세율구분코드'].str.slice(start = 0, stop = 1)\n",
    "        dataframe['단위무게(KG)가격'] = (dataframe['과세가격원화금액']/dataframe['신고중량(KG)']).map(lambda x: np.round(x, 0)).map(str)\n",
    "        dataframe.drop('HS10단위부호',axis=1,inplace=True)\n",
    "        dataframe['세금'] = (dataframe['과세가격원화금액']*dataframe['관세율']/dataframe['신고중량(KG)']/100.0)\n",
    "        numeric_features = ['신고중량(KG)', '과세가격원화금액','관세율']\n",
    "        dataframe.fillna('Missing', inplace=True)\n",
    "        for column in numeric_features:\n",
    "            dataframe[column] = rescaler(dataframe.pop(column).to_numpy())\n",
    "        np_data = dataframe[['신고중량(KG)', '과세가격원화금액','관세율']].to_numpy()\n",
    "        dataframe.drop(['신고일자','신고번호','우범여부','핵심적발' ,'수입자부호'],axis=1,inplace=True,errors='ignore')#,'HS10단위부호'\n",
    "        len_df = len(dataframe.index)\n",
    "        \n",
    "        add_count_ratio_list=['crime_count','priority_count','total_count']\n",
    "        reg_count_ratio_list=['crime_count','priority_count','total_count']\n",
    "        np_encoding=None\n",
    "        print(\"Before crime transform shape\",dataframe.shape)\n",
    "        for i,column in enumerate(categorical_features):\n",
    "            if column not in dataframe.columns:\n",
    "                continue\n",
    "            dataframe[column] = dataframe[column].map(str)\n",
    "            dict_col = self.mapping_dict[column]\n",
    "            np_count_ratio = np.zeros((len_df,len(add_count_ratio_list)))\n",
    "            if column in ['HS_upper','특송업체부호']:\n",
    "                np_ratio_addition = np.zeros((len_df,1))\n",
    "                for row in dataframe[column].index: \n",
    "                    val_data = dataframe[column][row]\n",
    "                    if val_data not in dict_col.keys():\n",
    "                        continue\n",
    "                    else:\n",
    "                        np_ratio_addition[row]=dict_col[val_data]['crime_ratio']\n",
    "                np_count_ratio=np.concatenate((np_count_ratio,np_ratio_addition),axis=1)\n",
    "            for row in dataframe[column].index: \n",
    "                val_data = dataframe[column][row]\n",
    "                # value you want to add\n",
    "                for idx, add_instance in enumerate(add_count_ratio_list):\n",
    "                    if val_data not in dict_col.keys():\n",
    "                        if add_instance in ['crime_ratio','priority_ratio'] : np_count_ratio[row][idx] = 0.0\n",
    "                        else : np_count_ratio[row][idx] = 0\n",
    "                    else :\n",
    "                        if (not dict_col[val_data]['is_crime_mask']) and (dict_col[val_data]['crime_ratio'] >= 0.3) :\n",
    "                            if add_instance != 'total_count':\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]*1.5\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]*1.5\n",
    "                            else :\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                        elif (not dict_col[val_data]['is_crime_mask']) and (dict_col[val_data]['crime_ratio'] < 0.3) :\n",
    "                            np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                            np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                        elif (dict_col[val_data]['is_crime_mask']) :\n",
    "                            if add_instance != 'total_count':\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]*0.5\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]*0.5\n",
    "                            else :\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                        else :\n",
    "                            np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                            np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "\n",
    "            # regularization\n",
    "            for idx,reg_instance in enumerate(reg_count_ratio_list):\n",
    "                np_count_ratio[:,idx] = (np_count_ratio[:,idx]-np_count_ratio[:,idx].mean())/(np_count_ratio[:,idx].var())\n",
    "            \n",
    "            if np_encoding is None: # 이진화 안하는 경우 ->위 주석처리 \n",
    "                np_data = np.concatenate((np_data,np_count_ratio, ), axis=1)\n",
    "            else: #이진화 하는 경우 -> 위 주석처리 x\n",
    "                np_encoding = np_encoding[:,::-1]\n",
    "                np_data = np.concatenate((np_data,np_count_ratio,np_encoding ), axis=1)\n",
    "\n",
    "            print('\\r[{}/{}] Finished Process'.format(i+1,len(categorical_features)),end='')\n",
    "                \n",
    "        print(\"After crime transform shape\",np_data.shape)\n",
    "        return np_data\n",
    "\n",
    "    def _priority_transform(self, dataframe:DataFrame)->np.ndarray:\n",
    "        rescaler=RescaleNumeric()\n",
    "        \"\"\"\n",
    "        categorical_features = ['통관지세관부호', '신고인부호', '수입자부호', '해외거래처부호', '특송업체부호', \n",
    "                                '수입통관계획코드', '수입신고구분코드', '수입거래구분코드', '수입종류코드', \n",
    "                                '징수형태코드', '운송수단유형코드', '반입보세구역부호', 'HS_upper', 'HS_middle', \n",
    "                                '적출국가코드', '원산지국가코드', '관세율구분코드']\n",
    "        \"\"\"\n",
    "        dataframe.fillna('Missing', inplace=True)\n",
    "        categorical_features = self.mapping_dict.keys()\n",
    "        dataframe['HS_upper'] = dataframe['HS10단위부호'] // 100000000 # 위 2자리\n",
    "        dataframe['HS_middle'] = dataframe['HS10단위부호'] // 1000000 # 위 4자리\n",
    "        dataframe['HS_low'] = dataframe['HS10단위부호'] // 10000 # 위 4자리\n",
    "        dataframe.drop('HS10단위부호',axis=1,inplace=True)\n",
    "        dataframe['관세율구분코드_1자리']=dataframe['관세율구분코드'].str.slice(start = 0, stop = 1)\n",
    "        dataframe['단위무게(KG)가격'] = (dataframe['과세가격원화금액']/dataframe['신고중량(KG)']).map(lambda x: np.round(x, 0)).map(str)\n",
    "        dataframe['세금'] = (dataframe['과세가격원화금액']*dataframe['관세율']/dataframe['신고중량(KG)']/100.0)\n",
    "        numeric_features = ['신고중량(KG)', '과세가격원화금액','관세율']\n",
    "        # numeric_features = ['신고중량(KG)', '과세가격원화금액','관세율']\n",
    "        for column in numeric_features:\n",
    "            dataframe[column] = rescaler(dataframe.pop(column).to_numpy())\n",
    "        np_data = dataframe[['신고중량(KG)', '과세가격원화금액','관세율']].to_numpy()\n",
    "        np_encoding=None\n",
    "        dataframe.drop(['신고일자','신고번호','우범여부','핵심적발', '수입자부호'],axis=1,inplace=True,errors='ignore')#,'HS10단위부호'\n",
    "        len_df = len(dataframe.index)\n",
    "        add_count_ratio_list=['priority_count','crime_count']\n",
    "        reg_count_ratio_list=['priority_count','crime_count']\n",
    "\n",
    "        print(\"Before priority transform shape\",dataframe.shape)\n",
    "        for i,column in enumerate(categorical_features):\n",
    "            if column not in dataframe.columns:\n",
    "                continue\n",
    "            dataframe[column] = dataframe[column].map(str)\n",
    "            dict_col = self.mapping_dict[column]\n",
    "            np_count_ratio = np.zeros((len_df,len(add_count_ratio_list)))\n",
    "            if column in []:\n",
    "                np_ratio_addition = np.zeros((len_df,1))\n",
    "                for row in dataframe[column].index: \n",
    "                    val_data = dataframe[column][row]\n",
    "                    if val_data not in dict_col.keys():\n",
    "                        continue\n",
    "                    else:\n",
    "                        np_ratio_addition[row]=dict_col[val_data]['priority_ratio']\n",
    "                np_count_ratio=np.concatenate((np_count_ratio,np_ratio_addition),axis=1)\n",
    "            for row in dataframe[column].index: \n",
    "                val_data = dataframe[column][row]\n",
    "\n",
    "                # value you want to add\n",
    "                for idx, add_instance in enumerate(add_count_ratio_list):\n",
    "                    if val_data not in dict_col.keys():\n",
    "                        if add_instance in ['crime_ratio','priority_ratio'] : np_count_ratio[row][idx] = 0.0\n",
    "                        else : np_count_ratio[row][idx] = 0\n",
    "                    else :\n",
    "                        if (not dict_col[val_data]['is_priority_mask']) and (dict_col[val_data]['priority_ratio'] >= 0.3) :\n",
    "                            if add_instance != 'crime_count':\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]*1.5\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]*1.5\n",
    "                            else :\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                        elif (not dict_col[val_data]['is_priority_mask']) and (dict_col[val_data]['priority_ratio'] < 0.3) :\n",
    "                            np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                            np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                        elif (dict_col[val_data]['is_priority_mask']) :\n",
    "                            if add_instance != 'crime_count':\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]*0.5\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]*0.5\n",
    "                            else :\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                                np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                        else :\n",
    "                            np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "                            np_count_ratio[row][idx] = dict_col[val_data][add_instance]\n",
    "\n",
    "            # regularization\n",
    "            for idx,reg_instance in enumerate(reg_count_ratio_list):\n",
    "                np_count_ratio[:,idx] = (np_count_ratio[:,idx]-np_count_ratio[:,idx].mean())/(np_count_ratio[:,idx].var())\n",
    "            \n",
    "            if np_encoding is None: # 이진화 안하는 경우 ->위 주석처리 \n",
    "                np_data = np.concatenate((np_data,np_count_ratio, ), axis=1)\n",
    "            else: #이진화 하는 경우 -> 위 주석처리 x\n",
    "                np_encoding = np_encoding[:,::-1]\n",
    "                np_data = np.concatenate((np_data,np_count_ratio,np_encoding ), axis=1)\n",
    "\n",
    "            print('\\r[{}/{}] Finished Process'.format(i+1,len(categorical_features)),end='')\n",
    "                \n",
    "        print(\"After priority transform shape\",np_data.shape)\n",
    "        return np_data\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 앞선 preprocessing이 완료된 numpy type의 데이터를 load하고, 학습에 이용되는 DataLoader를 생성하는 함수입니다.\n",
    "\n",
    "def load_dataset(data_path,configs):\n",
    "    # get data\n",
    "    if configs['preprocess']==True:\n",
    "        # npy_dict=get_data(data_path,configs)\n",
    "        preprocessing=Preprocessing(data_path,configs)\n",
    "        npy_dict=preprocessing.run()\n",
    "    if 'sj' in configs['mode']:\n",
    "        train_data=np.load(os.path.join(data_path,'sj_train_data.npy'))\n",
    "        train_targets=np.load(os.path.join(data_path,'sj_train_targets.npy'))\n",
    "        valid_data=np.load(os.path.join(data_path,'sj_valid_data.npy'))\n",
    "        valid_targets=np.load(os.path.join(data_path,'sj_valid_targets.npy'))\n",
    "\n",
    "        train_dataset=TensorDataset(train_data,train_targets)\n",
    "        valid_dataset=TensorDataset(valid_data,valid_targets)\n",
    "        return train_dataset,valid_dataset\n",
    "    else:    \n",
    "        npy_dict={\n",
    "            'train_crime_data':np.load(os.path.join(data_path,'train_crime_data.npy')),\n",
    "            'train_priority_data':np.load(os.path.join(data_path,'train_priority_data.npy')),\n",
    "            'crime_targets':np.load(os.path.join(data_path,'crime_targets.npy')), # y_1 (0,1)\n",
    "            'priority_targets':np.load(os.path.join(data_path,'priority_targets.npy')), #y_2 (0,1,2)\n",
    "            'train_indices':np.load(os.path.join(data_path,'train_indices.npy')),\n",
    "            'valid_indices':np.load(os.path.join(data_path,'valid_indices.npy')),\n",
    "            'test_crime_data':np.load(os.path.join(data_path,'test_crime_data.npy')),\n",
    "            'test_priority_data':np.load(os.path.join(data_path,'test_priority_data.npy')),\n",
    "        }\n",
    "    \n",
    "    if configs['mode'] in ['record','tsne_mixed','tsne_priority','tsne_crime']:\n",
    "        return npy_dict\n",
    "    #type casting\n",
    "    if 'xgboost' not in configs['mode']:\n",
    "        for key in npy_dict.keys():\n",
    "            npy_dict[key]=torch.from_numpy(npy_dict[key])\n",
    "            if npy_dict[key].dtype in [torch.float64,torch.double]:\n",
    "                npy_dict[key]=npy_dict[key].float()\n",
    "\n",
    "    #data separation\n",
    "        if 'crime' in configs['mode']:\n",
    "            crime_dataset=TensorDataset(npy_dict['train_crime_data'],npy_dict['crime_targets'])\n",
    "            #crime\n",
    "            train_dataset=Subset(crime_dataset,npy_dict['train_indices'])\n",
    "            valid_dataset=Subset(crime_dataset,npy_dict['valid_indices'])\n",
    "\n",
    "        elif 'priority' in configs['mode']:\n",
    "            #priority\n",
    "            train_data=npy_dict['train_priority_data'][npy_dict['train_indices'].long()]\n",
    "            valid_data=npy_dict['train_priority_data'][npy_dict['valid_indices'].long()]\n",
    "\n",
    "            train_target=npy_dict['priority_targets'][npy_dict['train_indices'].long()]\n",
    "            valid_target=npy_dict['priority_targets'][npy_dict['valid_indices'].long()]\n",
    "\n",
    "            # select 1 and 2\n",
    "            train_data=train_data[torch.logical_or(train_target==1,train_target==2)]\n",
    "            valid_data=valid_data[torch.logical_or(valid_target==1,valid_target==2)]\n",
    "            train_target=train_target[torch.logical_or(train_target==1,train_target==2)]\n",
    "            valid_target=valid_target[torch.logical_or(valid_target==1,valid_target==2)]\n",
    "\n",
    "            # change target 1 to 0, 2 to 1\n",
    "            train_target[train_target==1]=0\n",
    "            train_target[train_target==2]=1\n",
    "            valid_target[valid_target==1]=0\n",
    "            valid_target[valid_target==2]=1\n",
    "            train_dataset=TensorDataset(train_data, train_target)\n",
    "            valid_dataset=TensorDataset(valid_data, valid_target)\n",
    "\n",
    "        elif configs['mode']=='train_mixed':\n",
    "            #mixed\n",
    "            mixed_dataset=TensorDataset(npy_dict['train_crime_data'],npy_dict['train_priority_data'],npy_dict['crime_targets'],npy_dict['priority_targets'])\n",
    "            train_dataset=Subset(mixed_dataset,npy_dict['train_indices'])\n",
    "            valid_dataset=Subset(mixed_dataset,npy_dict['valid_indices'])\n",
    "\n",
    "        else:\n",
    "            print('No dataset')\n",
    "            raise NotImplementedError\n",
    "        return train_dataset, valid_dataset\n",
    "\n",
    "    else: #XGBOOST\n",
    "        if 'crime' in configs['mode']:\n",
    "            train_data=npy_dict['train_crime_data'][npy_dict['train_indices']]\n",
    "            valid_data=npy_dict['train_crime_data'][npy_dict['valid_indices']]\n",
    "\n",
    "            train_target=npy_dict['crime_targets'][npy_dict['train_indices']]\n",
    "            valid_target=npy_dict['crime_targets'][npy_dict['valid_indices']]\n",
    "\n",
    "        elif 'priority' in configs['mode']:\n",
    "            train_data=npy_dict['train_priority_data'][npy_dict['train_indices']]\n",
    "            valid_data=npy_dict['train_priority_data'][npy_dict['valid_indices']]\n",
    "\n",
    "            train_target=npy_dict['priority_targets'][npy_dict['train_indices']]\n",
    "            valid_target=npy_dict['priority_targets'][npy_dict['valid_indices']]\n",
    "\n",
    "        return train_data, train_target, valid_data, valid_target\n",
    "        \n",
    "\n",
    "def load_dataloader(data_path,configs):\n",
    "    sampler=None\n",
    "    if sampler is None:\n",
    "        shuffle=True\n",
    "    else:\n",
    "        shuffle=False\n",
    "    if 'xgboost' not in configs['mode']:\n",
    "        print('-------------load_dataloader--------------')\n",
    "        \n",
    "        train_dataset,valid_dataset=load_dataset(data_path,configs)\n",
    "        train_dataloader=DataLoader(train_dataset,batch_size=configs['batch_size'],num_workers=configs['num_workers'], sampler = sampler,shuffle=shuffle)\n",
    "        valid_dataloader=DataLoader(valid_dataset,batch_size=configs['batch_size'],num_workers=configs['num_workers'], shuffle = False)\n",
    "        return train_dataloader,valid_dataloader\n",
    "    else: #xgboost\n",
    "        train_data,train_target,valid_data,valid_target = load_dataset(data_path,configs)       \n",
    "        return train_data,train_target,valid_data,valid_target\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 우범여부와 핵심적발을 학습하는 딥러닝 모델입니다.\n",
    "# 우범여부를 학습하기 위한 모델인 CrimeModel과 핵심적발의 학습을 위한 모델인 PriorityModel이 존재합니다.\n",
    "# MixedModel은 앞의 CrimeModel과 PriorityModel을 통합한 모델로써, CrimeModel을 이용하여 우범여부를 학습 및 추론하고, 그 결과 및 PriorityModel을 이용하여 학심적발을 학습 및 추론합니다.\n",
    "\n",
    "class CrimeModel(nn.Module):\n",
    "    def __init__(self,input_space,output_space,configs):\n",
    "        super(CrimeModel,self).__init__()\n",
    "        self.model=nn.Sequential(\n",
    "            nn.Linear(input_space,5000),\n",
    "            nn.BatchNorm1d(5000),#5000),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(5000,1000),\n",
    "            nn.BatchNorm1d(1000),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(1000,300),\n",
    "            nn.BatchNorm1d(300),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(300,output_space)\n",
    "        )\n",
    "        self.weight1 = torch.tensor([0.2988, 1.0])\n",
    "        self.criterion=nn.CrossEntropyLoss(weight=self.weight1)\n",
    "        self.optimizer=torch.optim.Adam(self.model.parameters(),lr=configs['lr'],weight_decay=configs['weight_decay'])\n",
    "        self.scheduler=torch.optim.lr_scheduler.StepLR(optimizer=self.optimizer,step_size=configs['lr_decay'], gamma=configs['lr_decay_rate'])\n",
    "\n",
    "        for m in self.modules():\n",
    "            if isinstance(m,(nn.Linear)):\n",
    "                nn.init.kaiming_uniform_(m.weight,nonlinearity='relu')\n",
    "            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n",
    "                nn.init.constant_(m.weight, 1)\n",
    "                nn.init.constant_(m.bias, 0)\n",
    "    def forward(self,x):\n",
    "        y=self.model(x)\n",
    "        return y\n",
    "\n",
    "\n",
    "    def save_model(self,epoch,score_dict):\n",
    "        dict_model={\n",
    "            'epoch':epoch,\n",
    "            'crime_model_state_dict':self.model.state_dict(),\n",
    "        }\n",
    "        dict_model.update(score_dict)\n",
    "        return dict_model\n",
    "\n",
    "    def load_model(self,dict_model):\n",
    "        self.model.load_state_dict(dict_model['crime_model_state_dict'])\n",
    "\n",
    "class PriorityModel(nn.Module):\n",
    "    def __init__(self,input_space,output_space,configs):\n",
    "        super(PriorityModel,self).__init__()\n",
    "        self.model=nn.Sequential(\n",
    "            nn.Linear(input_space,5000),\n",
    "            nn.BatchNorm1d(5000),#5000),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(5000,1000),\n",
    "            nn.BatchNorm1d(1000),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(1000,300),\n",
    "            nn.BatchNorm1d(300),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(300,output_space)\n",
    "        )\n",
    "        self.weight2 = torch.tensor([1.0, 0.9431])\n",
    "        self.criterion=nn.CrossEntropyLoss(weight=self.weight2)\n",
    "        self.optimizer=torch.optim.Adam(self.model.parameters(),lr=configs['lr'],weight_decay=configs['weight_decay'])\n",
    "        self.scheduler=torch.optim.lr_scheduler.StepLR(optimizer=self.optimizer,step_size=configs['lr_decay'], gamma=configs['lr_decay_rate'])\n",
    "\n",
    "        for m in self.modules():\n",
    "            if isinstance(m,(nn.Linear)):\n",
    "                nn.init.kaiming_uniform_(m.weight,nonlinearity='relu')\n",
    "            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n",
    "                nn.init.constant_(m.weight, 1)\n",
    "                nn.init.constant_(m.bias, 0)\n",
    "    def forward(self,x):\n",
    "        y=self.model(x)\n",
    "        return y\n",
    "\n",
    "    def save_model(self,epoch,score_dict):\n",
    "        dict_model={\n",
    "            'epoch':epoch,\n",
    "            'priority_model_state_dict':self.model.state_dict(),\n",
    "        }\n",
    "        dict_model.update(score_dict)\n",
    "        return dict_model\n",
    "\n",
    "    def load_model(self,dict_model):\n",
    "        self.load_state_dict(dict_model['priority_model_state_dict'])\n",
    "\n",
    "## Mixed Model ##\n",
    "\n",
    "class MixedScheduler():\n",
    "    def __init__(self,crime_scheduler,priority_scheduler):\n",
    "        self.crime_scheduler=crime_scheduler\n",
    "        self.priority_scheduler=priority_scheduler\n",
    "\n",
    "    def step(self):\n",
    "        self.crime_scheduler.step()\n",
    "        self.priority_scheduler.step()\n",
    "\n",
    "class MixedOptimizer():\n",
    "    def __init__(self,crime_optim,priority_optim):\n",
    "        self.crime_optim=crime_optim\n",
    "        self.priority_optim=priority_optim\n",
    "        self.param_groups=self.crime_optim.param_groups\n",
    "    \n",
    "    def step(self):\n",
    "        self.crime_optim.step()\n",
    "        self.priority_optim.step()\n",
    "\n",
    "    def zero_grad(self):\n",
    "        self.priority_optim.zero_grad()\n",
    "        self.crime_optim.zero_grad()\n",
    "\n",
    "class MixedLossFunction():\n",
    "    def __init__(self,crime_criterion,priority_criterion,configs):\n",
    "        self.crime_criterion=crime_criterion\n",
    "        self.priority_criterion=priority_criterion\n",
    "        self.configs=configs\n",
    "        if self.configs['custom_loss']=='kd_loss':\n",
    "            self.custom_criterion=KDRegLoss(configs)\n",
    "        elif self.configs['custom_loss']=='fbeta_loss':\n",
    "            self.custom_criterion=FBetaLoss(configs)\n",
    "        else: \n",
    "            self.custom_criterion=None\n",
    "    \n",
    "    def __call__(self,crime_y_pred,priority_y_pred,crime_y_truth,priority_y_truth):\n",
    "        crime_custom_loss=0.0\n",
    "        priority_custom_loss=0.0\n",
    "\n",
    "        crime_loss=self.crime_criterion(crime_y_pred,crime_y_truth)\n",
    "        if self.custom_criterion is not None:\n",
    "            crime_custom_loss=self.custom_criterion(crime_y_pred,crime_y_truth)\n",
    "        crime_loss+=crime_custom_loss\n",
    "        \n",
    "        # search only\n",
    "        idx=torch.logical_or(priority_y_truth==1,priority_y_truth==2)\n",
    "        priority_y_truth=priority_y_truth[idx]-1\n",
    "        priority_y_pred=priority_y_pred[torch.stack((idx,idx),dim=1)].view(-1,2)\n",
    "        priority_loss=torch.nan_to_num(self.priority_criterion(priority_y_pred,priority_y_truth))\n",
    "        if self.custom_criterion is not None:\n",
    "            priority_custom_loss=torch.nan_to_num(self.custom_criterion(priority_y_pred,priority_y_truth))\n",
    "        priority_loss+=priority_custom_loss\n",
    "\n",
    "        return crime_loss,priority_loss\n",
    "\n",
    "class MixedModel:\n",
    "    def __init__(self,input_space,output_space,configs):\n",
    "        self.crime_model=CrimeModel(input_space[0],output_space,configs)\n",
    "        self.priority_model=PriorityModel(input_space[1],output_space,configs)#\n",
    "        self.criterion=MixedLossFunction(self.crime_model.criterion,self.priority_model.criterion,configs)\n",
    "        self.optimizer=MixedOptimizer(self.crime_model.optimizer,self.priority_model.optimizer)\n",
    "        self.scheduler=MixedScheduler(self.crime_model.scheduler,self.priority_model.scheduler)\n",
    "    \n",
    "    def __call__(self,crime_x,priority_x):\n",
    "        crime_output,priority_output=self.forward(crime_x,priority_x)\n",
    "        return crime_output,priority_output\n",
    "        \n",
    "\n",
    "    def forward(self,crime_x,priority_x):\n",
    "        crime_output=self.crime_model(crime_x)\n",
    "        softened_crime_output=f.softmax(crime_output,dim=1).detach().clone()\n",
    "        priority_output=self.priority_model(priority_x)\n",
    "        return crime_output,priority_output\n",
    "\n",
    "    def save_model(self,epoch,score_dict):\n",
    "        dict_model={\n",
    "            'epoch':epoch,\n",
    "            'crime_model_state_dict':self.crime_model.state_dict(),\n",
    "            'priority_model_state_dict':self.priority_model.state_dict(),\n",
    "        }\n",
    "        dict_model.update(score_dict)\n",
    "        return dict_model\n",
    "\n",
    "    def load_model(self, dict_model):\n",
    "        self.crime_model.load_state_dict(dict_model['crime_model_state_dict'])\n",
    "        self.priority_model.load_state_dict(dict_model['priority_model_state_dict'])\n",
    "    \n",
    "    def to(self,device):\n",
    "        self.crime_model.to(device)\n",
    "        self.priority_model.to(device)\n",
    "    \n",
    "    def train(self):\n",
    "        self.crime_model.train()\n",
    "        self.priority_model.train()\n",
    "\n",
    "    def eval(self):\n",
    "        self.crime_model.eval()\n",
    "        self.priority_model.eval()\n",
    "\n",
    "MODEL={\n",
    "    'crime':CrimeModel,\n",
    "    'priority':PriorityModel,\n",
    "    'mixed':MixedModel,\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 학습 성능을 개선하기 위한 Custom loss에 대한 Class입니다.\n",
    "# 최종 제출본을 학습한 경우에는 FBetaLoss가 사용되었습니다.\n",
    "\n",
    "class KDRegLoss():\n",
    "    def __init__(self,configs):\n",
    "        self.configs=configs\n",
    "        self.T=self.configs['temperature']# 20\n",
    "        self.alpha=self.configs['alpha']# 0.6\n",
    "    def __call__(self,outputs,labels):\n",
    "\n",
    "        \"\"\"\n",
    "        loss function for mannually-designed regularization: Tf-KD_{reg}\n",
    "        \"\"\"\n",
    "        self.alpha = 0.6\n",
    "        self.T = 20.0\n",
    "        correct_prob = 0.99    # the probability for correct class in u(k)\n",
    "        K = outputs.size(1)\n",
    "\n",
    "        teacher_soft = torch.ones_like(outputs)\n",
    "        teacher_soft = teacher_soft*(1-correct_prob)/(K-1)  # p^d(k)\n",
    "        for i in range(outputs.shape[0]):\n",
    "            teacher_soft[i ,labels[i]] = correct_prob\n",
    "        loss_soft_regu = nn.KLDivLoss(reduction='batchmean')(f.log_softmax(outputs, dim=1), f.softmax(teacher_soft/self.T, dim=1))*100\n",
    "\n",
    "        KD_loss = self.alpha*loss_soft_regu\n",
    "\n",
    "        return KD_loss\n",
    "\n",
    "class FBetaLoss():\n",
    "    def __init__(self,configs):\n",
    "        self.configs=configs\n",
    "        self.gae=self.configs['lambda']# 1.0\n",
    "        self.beta=self.configs['beta']# 10\n",
    "\n",
    "    def __call__(self,y_pred, y_true):\n",
    "        assert y_pred.ndim == 2\n",
    "        assert y_true.ndim == 1\n",
    "        epsilon=1e-7\n",
    "        y_true = f.one_hot(y_true, 2).to(torch.float32)\n",
    "        y_pred = f.softmax(y_pred, dim=1)\n",
    "        \n",
    "        tp = (y_true * y_pred).sum(dim=0).to(torch.float32)\n",
    "        tn = ((1 - y_true) * (1 - y_pred)).sum(dim=0).to(torch.float32)\n",
    "        fp = ((1 - y_true) * y_pred).sum(dim=0).to(torch.float32)\n",
    "        fn = (y_true * (1 - y_pred)).sum(dim=0).to(torch.float32) \n",
    "\n",
    "        precision = tp / (tp + fp + epsilon)\n",
    "        recall = tp / (tp + fn + epsilon)\n",
    "\n",
    "        f1 = (1+self.beta**2)* (precision*recall) / (self.beta**2*precision + recall + 1e-7)\n",
    "        f1 = f1.clamp(min=epsilon, max=1-epsilon)\n",
    "        return self.gae*(1 - f1.mean())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 모델의 학습 및 성능 추론을 위한 기능을 하는 Class입니다.\n",
    "# 기본 BaseLearner Class를 상속받아 CrimeLearner와 PriorityLearner를 사용합니다.\n",
    "\n",
    "class BaseLearner:\n",
    "    def __init__(self,logger,time_data, data_path, save_path, device, configs):\n",
    "        self.data_path=data_path # data load\n",
    "        self.save_path=save_path # save (tensorboard)\n",
    "        self.device=device # cuda\n",
    "        self.configs=configs\n",
    "        self.logger=logger\n",
    "        self.time_data=time_data\n",
    "\n",
    "class TorchLearner(BaseLearner):\n",
    "    def __init__(self, logger,time_data, data_path, save_path, device, configs):\n",
    "        super(TorchLearner,self).__init__(logger,time_data, data_path, save_path, device, configs)\n",
    "        self.train_dataloader,self.test_dataloader=load_dataloader(self.data_path,configs) # dataloader output(tensor) -> .numpy()\n",
    "        if 'crime' in configs['mode']:\n",
    "            self.input_space=self.train_dataloader.dataset[0][0].size()[0]\n",
    "            self.output_space=2\n",
    "        elif 'priority' in configs['mode']:\n",
    "            self.input_space=self.train_dataloader.dataset[0][0].size()[0]\n",
    "            self.output_space=2\n",
    "        else: #mixed\n",
    "            self.input_space=[self.train_dataloader.dataset[0][0].size()[0], self.train_dataloader.dataset[0][1].size()[0]]\n",
    "            self.output_space=2\n",
    "        self.model=MODEL[configs['mode'].split('_')[1]](self.input_space,self.output_space,configs)\n",
    "        self.criterion=self.model.criterion\n",
    "        self.optimizer=self.model.optimizer\n",
    "        self.scheduler=self.model.scheduler\n",
    "        self.logWriter=SummaryWriter(os.path.join(self.save_path,time_data))\n",
    "        self.score_dict={\n",
    "        'accuracy':0.0,\n",
    "        'total':0.0,\n",
    "        'precision':0.0,\n",
    "        'recall':0.0,\n",
    "        'f1score':0.0,\n",
    "        'loss':0.0,\n",
    "        'custom_loss':0.0,\n",
    "        }\n",
    "        self.best_f1score=0.0\n",
    "        self.best_acc=0.0\n",
    "        self.metric=dict()\n",
    "        if self.configs['custom_loss']=='kd_loss':\n",
    "            self.custom_criterion=KDRegLoss(configs)\n",
    "        elif self.configs['custom_loss']=='fbeta_loss':\n",
    "            self.custom_criterion=FBetaLoss(configs)\n",
    "        else: \n",
    "            self.custom_criterion=None\n",
    "\n",
    "    def run(self):\n",
    "        self.model.to(self.configs['device'])\n",
    "        self.logger.info(self.configs)\n",
    "\n",
    "        for epoch in range(1,self.configs['epochs']+1):\n",
    "            train_score_dict=copy.deepcopy(self.score_dict)\n",
    "            eval_score_dict=copy.deepcopy(self.score_dict)\n",
    "            #Init\n",
    "\n",
    "            #Train\n",
    "            self.metric=dict()# metric\n",
    "            print('='*30)\n",
    "            train_tik=time.time()\n",
    "            train_score_dict=self._train(epoch,train_score_dict)\n",
    "            train_tok=time.time()\n",
    "            print('\\n Learning Rate: {:.8f} Learning Time: {:.3f}s'.format(self.optimizer.param_groups[0]['lr'],train_tok-train_tik))\n",
    "            self._epoch_end_logger(epoch,train_score_dict,'train')\n",
    "\n",
    "            #Eval\n",
    "            self.metric=dict()\n",
    "            eval_score_dict=self._eval(epoch,eval_score_dict)\n",
    "            self._epoch_end_logger(epoch,eval_score_dict,'eval')\n",
    "\n",
    "            self.scheduler.step()\n",
    "        \n",
    "        self.logger = logging.getLogger('best')\n",
    "        self.logger.info('[Mode {}] [Best Acc {:.2f}] [Best F1 {:.3f}]'.format(self.configs['mode'],self.best_acc,self.best_f1score))\n",
    "\n",
    "        print('==End==')\n",
    "\n",
    "    def _train(self,epoch,score_dict):\n",
    "        self.model.train()\n",
    "        train_loss=0.0\n",
    "        train_custom_loss=0.0\n",
    "        custom_loss=None\n",
    "\n",
    "        for batch_idx, (data,targets) in enumerate(self.train_dataloader):\n",
    "            data,targets=data.to(self.configs['device']),targets.to(self.configs['device'])\n",
    "\n",
    "            outputs=self.model(data)\n",
    "            if self.custom_criterion is None:\n",
    "                loss=self.criterion(outputs,targets)\n",
    "            else:\n",
    "                loss=self.criterion(outputs,targets)\n",
    "                custom_loss=self.custom_criterion(outputs,targets)\n",
    "                train_custom_loss+=custom_loss.item()\n",
    "                loss+=custom_loss\n",
    "\n",
    "            self.optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            self.optimizer.step()\n",
    "            train_loss+=loss.item()\n",
    "            # predictions=torch.round(outputs).view(-1)#linear regression\n",
    "            predictions=torch.max(outputs,dim=1)[1].clone()\n",
    "            if self.metric == dict():\n",
    "                self.metric['predictions']=predictions\n",
    "                self.metric['targets']=targets\n",
    "            else:\n",
    "                self.metric['predictions']=torch.cat((self.metric['predictions'],predictions),dim=0)\n",
    "                self.metric['targets']=torch.cat((self.metric['targets'],targets),dim=0)\n",
    "            if batch_idx%50==1:\n",
    "                acc=(self.metric['predictions']==self.metric['targets']).sum()/self.metric['targets'].size(0)*100.0\n",
    "                print('\\r{}epoch {}/{}, [Acc] {:.2f} [Loss] {:.5f}'.format(epoch,self.metric['targets'].size(0),\n",
    "                len(self.train_dataloader.dataset),acc,train_loss/(batch_idx+1)),end='')\n",
    "            \n",
    "        score_dict=self._get_score(self.metric['predictions'],self.metric['targets'],score_dict)\n",
    "        score_dict['loss']=train_loss/(batch_idx+1)\n",
    "        if custom_loss is not None:\n",
    "            score_dict['custom_loss']=train_custom_loss/(batch_idx+1)\n",
    "        return score_dict\n",
    "\n",
    "  \n",
    "    def _eval(self,epoch,score_dict):\n",
    "        self.model.eval()\n",
    "        eval_loss=0.0\n",
    "        eval_custom_loss=0.0\n",
    "        custom_loss=None\n",
    "\n",
    "        with torch.no_grad():\n",
    "            for batch_idx,(data,targets) in enumerate(self.test_dataloader):\n",
    "                data,targets=data.to(self.configs['device']),targets.to(self.configs['device'])\n",
    "\n",
    "                outputs=self.model(data)\n",
    "                predictions=torch.max(outputs,dim=1)[1].clone() # cross-entropy\n",
    "                # predictions=torch.round(outputs).view(-1)#linear regression\n",
    "                if self.custom_criterion is None:\n",
    "                    loss=self.criterion(outputs,targets)\n",
    "                else:\n",
    "                    loss=self.criterion(outputs,targets)\n",
    "                    custom_loss=self.custom_criterion(outputs,targets)\n",
    "                    eval_custom_loss+=custom_loss.item()\n",
    "                    loss+=custom_loss\n",
    "\n",
    "                eval_loss +=loss.item()\n",
    "                if self.metric == dict():\n",
    "                    self.metric['predictions']=predictions\n",
    "                    self.metric['targets']=targets\n",
    "                else:\n",
    "                    self.metric['predictions']=torch.cat((self.metric['predictions'],predictions),dim=0)\n",
    "                    self.metric['targets']=torch.cat((self.metric['targets'],targets),dim=0)\n",
    "        score_dict=self._get_score(self.metric['predictions'],self.metric['targets'],score_dict)\n",
    "        score_dict['loss']=eval_loss/(batch_idx+1)\n",
    "        if custom_loss is not None:\n",
    "            score_dict['custom_loss']=eval_custom_loss/(batch_idx+1)\n",
    "\n",
    "        return score_dict\n",
    "\n",
    "    def save_models(self,epoch, score_dict,model_type):\n",
    "        dict_model= self.model.save_model(epoch,score_dict)\n",
    "        torch.save(dict_model,\n",
    "        os.path.join(self.save_path,self.time_data,'best_{}_model.pt'.format(model_type)))\n",
    "\n",
    "    def load_model(self):\n",
    "        dict_model=torch.load(self.save_path,self.configs['file_name'],'best_{}_model.pt'.format(self.configs['mode'].split('_')[1]))\n",
    "        self.model.load_model(dict_model)\n",
    "\n",
    "    def _get_score(self,predictions,targets,score_dict):\n",
    "        score_dict=calc_score(predictions,targets,score_dict)\n",
    "        return score_dict\n",
    "        \n",
    "    def _epoch_end_logger(self,epoch,score_dict,mode='train'):\n",
    "        if mode=='eval':\n",
    "            if self.best_f1score<score_dict['f1score']:\n",
    "                self.best_f1score=score_dict['f1score']\n",
    "                self.best_acc=score_dict['accuracy']\n",
    "                self.save_models(epoch,score_dict,self.configs['mode'].split('_')[1])\n",
    "        self._write_logger(epoch,self.configs['mode'].split('_')[1],score_dict,mode)\n",
    "\n",
    "    def _write_logger(self,epoch,model_type,score_dict,mode):\n",
    "        self.logger=logging.getLogger('{}'.format(mode))\n",
    "        if score_dict['custom_loss']==0.0:\n",
    "            #self.logger.info('\\n[{} Epoch {}] [loss] {:.5f} [acc] {:.2f} [precision] {:.2f} [recall] {:.2f} [f1score] {:.2f}'.format(\n",
    "            #    epoch,model_type,score_dict['loss'], score_dict['accuracy'],score_dict['precision'],score_dict['recall'],score_dict['f1score']))\n",
    "            print('{} {} is completed'.format(model_type, mode))\n",
    "        else:\n",
    "            self.logger.info('\\n[{} Epoch {}] [ce loss] {:.5f} [custom loss] {:.5f} [acc] {:.2f} [precision] {:.2f} [recall] {:.2f} [f1score] {:.2f}'.format(\n",
    "                epoch,model_type,score_dict['loss'],score_dict['custom_loss'], score_dict['accuracy'],score_dict['precision'],score_dict['recall'],score_dict['f1score']))\n",
    "        self.logWriter.add_scalars('{}_{}'.format(mode,model_type),score_dict,epoch)\n",
    "    \n",
    "\n",
    "class CrimeLearner(TorchLearner):\n",
    "    def __init__(self,logger,imte_data, data_path, save_path, device, configs):\n",
    "        super(CrimeLearner,self).__init__(logger,imte_data, data_path, save_path, device, configs)\n",
    "        \n",
    "class PriorityLearner(TorchLearner):\n",
    "    def __init__(self,logger,imte_data, data_path, save_path, device, configs):\n",
    "        super(PriorityLearner,self).__init__(logger,imte_data, data_path, save_path, device, configs)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 모델의 학습 및 성능 추론을 위한 기능을 하는 Class입니다.\n",
    "# 기본 BaseLearner Class를 상속받아 CrimeLearner와 PriorityLearner를 사용합니다.\n",
    "\n",
    "class MixedLearner(TorchLearner):\n",
    "    def __init__(self, logger:logging,time_data, data_path, save_path, device, configs):\n",
    "        super(MixedLearner,self).__init__(logger,time_data, data_path, save_path, device, configs)\n",
    "        self.score_dict={\n",
    "            'loss':0.0,\n",
    "            'total':0.0,\n",
    "            'crime':copy.deepcopy(self.score_dict),\n",
    "            'priority':copy.deepcopy(self.score_dict),\n",
    "            'advantage':0.0\n",
    "        }\n",
    "        self.best_f1score={'crime':0.0,'priority':0.0}\n",
    "        self.best_acc={'crime':0.0,'priority':0.0}\n",
    "        self.best_epoch=0\n",
    "        self.best_advantage=0.0\n",
    "\n",
    "\n",
    "    def eval_run(self):\n",
    "        self.model.to(self.configs['device'])\n",
    "        self.logger.info(self.configs)\n",
    "\n",
    "\n",
    "        crime_dict=torch.load('C:\\\\anaconda3\\envs\\\\torch\\Customs_Kaggle\\crime-cargo-detection\\\\training_data\\\\07-16_15-13-31\\\\best_crime_model.pt')\n",
    "        priority_dict=torch.load('C:\\\\anaconda3\\envs\\\\torch\\Customs_Kaggle\\crime-cargo-detection\\\\training_data\\\\07-16_15-13-31\\\\best_priority_model.pt')\n",
    "        print(\"========== Performances ==========\")\n",
    "        #print(\"crime F1: {:.3f} crime Acc: {:.3f}\".format(crime_dict['f1score'],crime_dict['accuracy']))\n",
    "        #print(\"priority F1: {:.3f} priority Acc: {:.3f}\".format(priority_dict['f1score'],priority_dict['accuracy']))\n",
    "        print(\"==================================\")\n",
    "        dict_model={**crime_dict,**priority_dict}\n",
    "        self.model.load_model(dict_model)\n",
    "            \n",
    "        print(\"Model Load Complete\")\n",
    "        self.model.to(self.configs['device'])\n",
    "\n",
    "        if 'mixed' not in self.configs['mode']:\n",
    "            metric=self.run_ind()\n",
    "        else: #mixed\n",
    "            metric=self.run_mix()\n",
    "        print(\"Prediction Complete\")\n",
    "\n",
    "    def run_mix(self):\n",
    "        self.model.eval()        \n",
    "        eval_score_dict=copy.deepcopy(self.score_dict)\n",
    "        print('start runmix')\n",
    "        self._eval(1, eval_score_dict)\n",
    "        print('finish runmix')\n",
    "\n",
    "    def run(self):\n",
    "        self.model.to(self.configs['device'])\n",
    "        self.logger.info(self.configs)\n",
    "\n",
    "        for epoch in range(1,self.configs['epochs']+1):\n",
    "            train_score_dict=copy.deepcopy(self.score_dict)\n",
    "            eval_score_dict=copy.deepcopy(self.score_dict)\n",
    "            #Init\n",
    "\n",
    "            #Train\n",
    "            self.metric=dict()# metric\n",
    "            print('='*30)\n",
    "            train_tik=time.time()\n",
    "            train_score_dict=self._train(epoch,train_score_dict)\n",
    "            train_tok=time.time()\n",
    "            print('\\n Learning Rate: {:.8f} Learning Time: {:.3f}s'.format(self.optimizer.param_groups[0]['lr'],train_tok-train_tik))\n",
    "            self._epoch_end_logger(epoch,train_score_dict,'train')\n",
    "\n",
    "            #Eval\n",
    "            self.metric=dict()\n",
    "            eval_metric={'Current':None,'Prior Best':None}\n",
    "            self.logger=logging.getLogger('Current')\n",
    "            eval_metric['Current']=self._eval(epoch,eval_score_dict)\n",
    "            #self.logger.info('[Current eval] Crime Acc: {:.2f} F1: {:.2f} Priority Acc: {:.2f} F1 {:.2f} [adv] {:.2f}'.format(eval_metric['Current']['crime']['accuracy'],eval_metric['Current']['crime']['f1score'],eval_metric['Current']['priority']['accuracy'],eval_metric['Current']['priority']['f1score'],eval_metric['Current']['advantage']))\n",
    "            if epoch>=2:\n",
    "                eval_score_dict=copy.deepcopy(self.score_dict)\n",
    "                self.save_tmp_model(epoch,eval_metric['Current']['crime'],'Current')\n",
    "                self.load_tmp_model('Prior Best')\n",
    "                self.logger=logging.getLogger('Prior Best')\n",
    "                eval_metric['Prior Best']=self._eval(epoch,eval_score_dict)\n",
    "                #self.logger.info('[Prior Best eval] Crime Acc: {:.2f} F1: {:.2f} Priority Acc: {:.2f} F1 {:.2f} [adv] {:.2f}'.format(eval_metric['Prior Best']['crime']['accuracy'],eval_metric['Prior Best']['crime']['f1score'],eval_metric['Prior Best']['priority']['accuracy'],eval_metric['Prior Best']['priority']['f1score'],eval_metric['Prior Best']['advantage']))\n",
    "                if eval_metric['Current']['advantage']>eval_metric['Prior Best']['advantage']:\n",
    "                    print('Current Advantage is Best')\n",
    "                    eval_score_dict=copy.deepcopy(eval_metric['Current'])\n",
    "                    self.load_tmp_model('Current')\n",
    "                    self.save_tmp_model(epoch,eval_score_dict['crime'],'Prior Best')\n",
    "                else:\n",
    "                    print('Prior Advantage is Best')\n",
    "                    eval_score_dict=copy.deepcopy(eval_metric['Prior Best'])\n",
    "            else:\n",
    "                eval_score_dict=copy.deepcopy(eval_metric['Current'])\n",
    "\n",
    "            self._epoch_end_logger(epoch,eval_score_dict,'eval')\n",
    "            if epoch>=2:\n",
    "                self.load_tmp_model('Current')\n",
    "            self.scheduler.step()\n",
    "        \n",
    "        self.logger = logging.getLogger('best')\n",
    "        self.logger.info('[Mode {}] [Best Crime Acc {:.2f}] [Best Crime F1 {:.3f}]'.format(self.configs['mode'],self.best_acc['crime'],self.best_f1score['crime']))\n",
    "        self.logger.info('[Mode {}] [Best Priority Acc {:.2f}] [Best Priority F1 {:.3f}]'.format(self.configs['mode'],self.best_acc['priority'],self.best_f1score['priority']))\n",
    "        self.logger.info('[Best Advantage Score] {}'.format(self.best_advantage))\n",
    "        print('==End==')    \n",
    "\n",
    "\n",
    "    def _train(self,epoch,score_dict):\n",
    "        self.model.train()\n",
    "        train_loss=0.0\n",
    "\n",
    "        for batch_idx, (crime_data, priority_data, crime_targets, priority_targets) in enumerate(self.train_dataloader):\n",
    "            crime_data, priority_data = crime_data.to(self.configs['device']), priority_data.to(self.configs['device'])\n",
    "            crime_targets, priority_targets = crime_targets.to(self.configs['device']), priority_targets.to(self.configs['device'])\n",
    "            crime_outputs,priority_outputs=self.model(crime_data, priority_data)\n",
    "            crime_loss,priority_loss=self.criterion(crime_outputs,priority_outputs,crime_targets,priority_targets)\n",
    "            loss=crime_loss+priority_loss\n",
    "\n",
    "            self.optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            self.optimizer.step()\n",
    "            crime_predictions=torch.max(crime_outputs,dim=1)[1].clone()\n",
    "            priority_predictions=torch.max(priority_outputs,dim=1)[1].clone()\n",
    "\n",
    "            #loss\n",
    "            score_dict['crime']['loss']+=crime_loss.item()\n",
    "            score_dict['priority']['loss']+=priority_loss.item()\n",
    "            priority_predictions[crime_predictions==0]=-1\n",
    "            self._save_score(crime_predictions,crime_targets,priority_predictions+1,priority_targets)\n",
    "            train_loss+=loss.item()\n",
    "            score_dict['total']+=crime_targets.size(0)\n",
    "            if batch_idx%50==1:\n",
    "                crime_acc=(self.metric['crime']['predictions']==self.metric['crime']['targets']).sum()/self.metric['crime']['targets'].size(0)*100.0\n",
    "                priority_acc=(self.metric['priority']['predictions']==self.metric['priority']['targets']).sum()/self.metric['priority']['targets'].size(0)*100.0\n",
    "                #print('\\r{}epoch {}/{}, [Crime Acc] {:.2f} [Priority Acc] {:.2f}  [Loss] {:.5f}'.format(epoch,int(score_dict['total']),\n",
    "                print('\\r{}epoch {}/{}'.format(epoch,int(score_dict['total']), len(self.train_dataloader.dataset)),end='')\n",
    "\n",
    "        # crime\n",
    "        score_dict['crime']=self._get_score(self.metric['crime']['predictions'],self.metric['crime']['targets'],score_dict['crime'])\n",
    "        # priority\n",
    "        score_dict['priority']=self._get_score(self.metric['priority']['predictions'],self.metric['priority']['targets'],score_dict['priority'])\n",
    "\n",
    "        score_dict['loss']=train_loss/(batch_idx+1)\n",
    "        score_dict['crime']['loss']=score_dict['crime']['loss']/(batch_idx+1)\n",
    "        score_dict['priority']['loss']=score_dict['priority']['loss']/(batch_idx+1)\n",
    "        score_dict['advantage'] = (score_dict['crime']['f1score'] + score_dict['priority']['f1score'])*0.5\n",
    "        return score_dict\n",
    "\n",
    "\n",
    "    def _eval(self,epoch,score_dict):\n",
    "\n",
    "        self.model.eval()\n",
    "        eval_loss=0.0\n",
    "\n",
    "        with torch.no_grad():\n",
    "            for batch_idx,(crime_data,priority_data,crime_targets,priority_targets) in enumerate(self.test_dataloader):\n",
    "                crime_data,priority_data=crime_data.to(self.configs['device']),priority_data.to(self.configs['device'])\n",
    "                crime_targets,priority_targets=crime_targets.to(self.configs['device']),priority_targets.to(self.configs['device'])\n",
    "                \n",
    "                crime_outputs,priority_outputs=self.model(crime_data,priority_data)\n",
    "                crime_loss,priority_loss=self.criterion(crime_outputs,priority_outputs,crime_targets,priority_targets)\n",
    "                crime_predictions=torch.max(crime_outputs,dim=1)[1].clone()\n",
    "                priority_predictions=torch.max(priority_outputs,dim=1)[1].clone()\n",
    "\n",
    "                #loss\n",
    "                loss=crime_loss+priority_loss\n",
    "                score_dict['crime']['loss']+=crime_loss.item()\n",
    "                score_dict['priority']['loss']+=priority_loss.item()\n",
    "                priority_predictions[crime_predictions==0]=-1\n",
    "                self._save_score(crime_predictions,crime_targets,priority_predictions+1,priority_targets)\n",
    "                eval_loss +=loss.item()\n",
    "        # crime\n",
    "        score_dict['crime']=self._get_score(self.metric['crime']['predictions'],self.metric['crime']['targets'],score_dict['crime'])\n",
    "        # priority\n",
    "        score_dict['priority']=self._get_score(self.metric['priority']['predictions'],self.metric['priority']['targets'],score_dict['priority'])\n",
    "                \n",
    "        score_dict['loss']=eval_loss/(batch_idx+1)\n",
    "        score_dict['crime']['loss']=score_dict['crime']['loss']/(batch_idx+1)\n",
    "        score_dict['priority']['loss']=score_dict['priority']['loss']/(batch_idx+1)\n",
    "        score_dict['advantage'] = (score_dict['crime']['f1score'] + score_dict['priority']['f1score'])*0.5\n",
    "        # print('[Crime] {0:.4f}\\t [Priority] {1:.4f}\\t [Total F1] {2:.4f}\\t'.format(score_dict['crime']['f1score'], score_dict['priority']['f1score'], score_dict['advantage']))\n",
    "\n",
    "        return score_dict\n",
    "\n",
    "    def _save_score(self,crime_predictions,crime_targets,priority_predictions,priority_targets):\n",
    "        if self.metric==dict():\n",
    "            self.metric['crime']=dict()\n",
    "            self.metric['priority']=dict()\n",
    "            self.metric['crime']['predictions']= crime_predictions\n",
    "            self.metric['crime']['targets']= crime_targets\n",
    "            self.metric['priority']['predictions']= priority_predictions\n",
    "            self.metric['priority']['targets']= priority_targets\n",
    "        else:\n",
    "            self.metric['crime']['predictions']=torch.cat((self.metric['crime']['predictions'],crime_predictions),dim=0)\n",
    "            self.metric['crime']['targets']=torch.cat((self.metric['crime']['targets'],crime_targets),dim=0)\n",
    "            self.metric['priority']['predictions']=torch.cat((self.metric['priority']['predictions'],priority_predictions),dim=0)\n",
    "            self.metric['priority']['targets']=torch.cat((self.metric['priority']['targets'],priority_targets),dim=0)\n",
    "            \n",
    "    def _epoch_end_logger(self,epoch,score_dict,mode='train'):\n",
    "        for model_type in ['crime','priority']:\n",
    "            this_score_dict=score_dict[model_type]\n",
    "            self._write_logger(epoch,model_type,this_score_dict,mode)\n",
    "\n",
    "        if mode=='eval':\n",
    "            score_dict['advantage']=(score_dict['crime']['f1score']+score_dict['priority']['f1score'])*0.5\n",
    "            self.logger.info(\"Advantage Score: {:.2f} Best Advantage Score: {:.2f} Best Epoch: {}\".format(score_dict['advantage'],self.best_advantage,self.best_epoch))\n",
    "            if epoch==1:\n",
    "                self.best_epoch=1\n",
    "                self.best_f1score[model_type]=score_dict[model_type]['f1score']\n",
    "                self.best_acc[model_type]=score_dict[model_type]['accuracy']\n",
    "                self.best_advantage=score_dict['advantage']\n",
    "                for model_type in ['crime','priority']:\n",
    "                    self.save_models(epoch,score_dict[model_type],model_type)\n",
    "            if self.best_advantage<(score_dict['crime']['f1score']+score_dict['priority']['f1score'])*0.5:\n",
    "                self.best_advantage=(score_dict['crime']['f1score']+score_dict['priority']['f1score'])*0.5\n",
    "                self.best_epoch=epoch\n",
    "                for model_type in ['crime','priority']:\n",
    "                    self.best_f1score[model_type]=score_dict[model_type]['f1score']\n",
    "                    self.best_acc[model_type]=score_dict[model_type]['accuracy']\n",
    "                    self.save_models(epoch,score_dict[model_type],model_type)\n",
    "    \n",
    "    def load_model(self):\n",
    "        crime_dict=torch.load(self.save_path,self.configs['file_name'],'best_crime_model.pt')\n",
    "        priority_dict=torch.load(self.save_path,self.configs['file_name'],'best_priority_model.pt')\n",
    "        dict_model={'crime':crime_dict,'priority':priority_dict}\n",
    "        print(\"Best Advantage: {:.2f}\".format((crime_dict['f1score']+priority_dict['f1score'])*0.5))\n",
    "        self.model.load_model(dict_model)\n",
    "        \n",
    "    def save_tmp_model(self,epoch,score_dict,current):\n",
    "        self.optimizer.zero_grad()\n",
    "        if current=='Prior Best':\n",
    "            word='best'\n",
    "        elif current=='Current':\n",
    "            word='tmp'\n",
    "        tmp_dict={\n",
    "            'crime_model_state_dict':self.model.crime_model.state_dict()}\n",
    "        save_dict={**tmp_dict,**score_dict}\n",
    "        torch.save(save_dict,os.path.join(self.save_path,self.time_data,'{}_crime_model.pt'.format(word)))\n",
    "\n",
    "    def load_tmp_model(self,current):\n",
    "        self.optimizer.zero_grad()\n",
    "        if current=='Prior Best':\n",
    "            word='best'\n",
    "        elif current=='Current':\n",
    "            word='tmp'\n",
    "        tmp_dict=torch.load(os.path.join(self.save_path,self.time_data,'{}_crime_model.pt'.format(word)))\n",
    "        self.model.crime_model.cpu()\n",
    "        self.model.crime_model.load_state_dict(tmp_dict['crime_model_state_dict'])\n",
    "        self.model.crime_model.to(self.configs['device'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------load_dataloader--------------\n",
      "[2021-07-21 12:18:08,441] [numexpr.utils] NumExpr defaulting to 8 threads.\n",
      "(71695, 21)\n",
      "calc count_ratio for only train\n",
      "(89619, 21)\n",
      "Generate Json complete\n",
      "Before priority transform shape (89619, 23)\n",
      "[21/21] Finished ProcessAfter priority transform shape (89619, 41)\n",
      "Before crime transform shape (89619, 23)\n",
      "[21/21] Finished ProcessAfter crime transform shape (89619, 62)\n",
      "Before priority transform shape (10273, 23)\n",
      "[21/21] Finished ProcessAfter priority transform shape (10273, 41)\n",
      "Before crime transform shape (10273, 23)\n",
      "[21/21] Finished ProcessAfter crime transform shape (10273, 62)\n",
      "[2021-07-21 12:18:51,787] [main] {'mode': 'train_mixed', 'seed': 11, 'batch_size': 256, 'device': 'cuda', 'colab': False, 'num_workers': 3, 'lr': 0.001, 'weight_decay': 0.0005, 'epochs': 50, 'lr_decay': 5, 'lr_decay_rate': 0.8, 'custom_loss': 'fbeta_loss', 'preprocess': True, 'split_dataset': False, 'lambda': 0.3, 'beta': 15.0, 'split_ratio': 0.8, 'only_train': True}\n",
      "==============================\n",
      "1epoch 64512/71695\n",
      " Learning Rate: 0.00100000 Learning Time: 8.381s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:19:01,675] [eval] Advantage Score: 42.57 Best Advantage Score: 0.00 Best Epoch: 0\n",
      "==============================\n",
      "2epoch 64512/71695\n",
      " Learning Rate: 0.00100000 Learning Time: 7.516s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Current Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:19:12,548] [eval] Advantage Score: 47.21 Best Advantage Score: 42.57 Best Epoch: 1\n",
      "==============================\n",
      "3epoch 64512/71695\n",
      " Learning Rate: 0.00100000 Learning Time: 7.516s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Current Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:19:23,469] [eval] Advantage Score: 47.64 Best Advantage Score: 47.21 Best Epoch: 2\n",
      "==============================\n",
      "4epoch 64512/71695\n",
      " Learning Rate: 0.00100000 Learning Time: 7.458s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Current Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:19:34,384] [eval] Advantage Score: 49.78 Best Advantage Score: 47.64 Best Epoch: 3\n",
      "==============================\n",
      "5epoch 64512/71695\n",
      " Learning Rate: 0.00100000 Learning Time: 7.488s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:19:45,254] [eval] Advantage Score: 48.76 Best Advantage Score: 49.78 Best Epoch: 4\n",
      "==============================\n",
      "6epoch 64512/71695\n",
      " Learning Rate: 0.00080000 Learning Time: 7.473s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:19:55,884] [eval] Advantage Score: 48.81 Best Advantage Score: 49.78 Best Epoch: 4\n",
      "==============================\n",
      "7epoch 64512/71695\n",
      " Learning Rate: 0.00080000 Learning Time: 7.534s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:20:06,576] [eval] Advantage Score: 47.91 Best Advantage Score: 49.78 Best Epoch: 4\n",
      "==============================\n",
      "8epoch 64512/71695\n",
      " Learning Rate: 0.00080000 Learning Time: 7.523s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:20:17,251] [eval] Advantage Score: 49.97 Best Advantage Score: 49.78 Best Epoch: 4\n",
      "==============================\n",
      "9epoch 64512/71695\n",
      " Learning Rate: 0.00080000 Learning Time: 7.518s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:20:28,044] [eval] Advantage Score: 47.48 Best Advantage Score: 49.97 Best Epoch: 8\n",
      "==============================\n",
      "10epoch 64512/71695\n",
      " Learning Rate: 0.00080000 Learning Time: 7.477s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:20:38,717] [eval] Advantage Score: 48.27 Best Advantage Score: 49.97 Best Epoch: 8\n",
      "==============================\n",
      "11epoch 64512/71695\n",
      " Learning Rate: 0.00064000 Learning Time: 7.500s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:20:49,379] [eval] Advantage Score: 47.36 Best Advantage Score: 49.97 Best Epoch: 8\n",
      "==============================\n",
      "12epoch 64512/71695\n",
      " Learning Rate: 0.00064000 Learning Time: 7.533s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:21:00,065] [eval] Advantage Score: 49.52 Best Advantage Score: 49.97 Best Epoch: 8\n",
      "==============================\n",
      "13epoch 64512/71695\n",
      " Learning Rate: 0.00064000 Learning Time: 7.504s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Current Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:21:10,807] [eval] Advantage Score: 50.25 Best Advantage Score: 49.97 Best Epoch: 8\n",
      "==============================\n",
      "14epoch 64512/71695\n",
      " Learning Rate: 0.00064000 Learning Time: 7.523s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Current Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:21:21,754] [eval] Advantage Score: 49.90 Best Advantage Score: 50.25 Best Epoch: 13\n",
      "==============================\n",
      "15epoch 64512/71695\n",
      " Learning Rate: 0.00064000 Learning Time: 7.519s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Current Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:21:32,453] [eval] Advantage Score: 50.80 Best Advantage Score: 50.25 Best Epoch: 13\n",
      "==============================\n",
      "16epoch 64512/71695\n",
      " Learning Rate: 0.00051200 Learning Time: 7.508s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:21:43,302] [eval] Advantage Score: 50.32 Best Advantage Score: 50.80 Best Epoch: 15\n",
      "==============================\n",
      "17epoch 64512/71695\n",
      " Learning Rate: 0.00051200 Learning Time: 7.489s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:21:53,958] [eval] Advantage Score: 47.35 Best Advantage Score: 50.80 Best Epoch: 15\n",
      "==============================\n",
      "18epoch 64512/71695\n",
      " Learning Rate: 0.00051200 Learning Time: 7.569s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:22:04,675] [eval] Advantage Score: 49.47 Best Advantage Score: 50.80 Best Epoch: 15\n",
      "==============================\n",
      "19epoch 64512/71695\n",
      " Learning Rate: 0.00051200 Learning Time: 7.532s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:22:15,368] [eval] Advantage Score: 49.51 Best Advantage Score: 50.80 Best Epoch: 15\n",
      "==============================\n",
      "20epoch 64512/71695\n",
      " Learning Rate: 0.00051200 Learning Time: 7.531s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:22:26,033] [eval] Advantage Score: 50.63 Best Advantage Score: 50.80 Best Epoch: 15\n",
      "==============================\n",
      "21epoch 64512/71695\n",
      " Learning Rate: 0.00040960 Learning Time: 7.544s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Current Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:22:36,820] [eval] Advantage Score: 50.78 Best Advantage Score: 50.80 Best Epoch: 15\n",
      "==============================\n",
      "22epoch 64512/71695\n",
      " Learning Rate: 0.00040960 Learning Time: 7.569s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:22:47,467] [eval] Advantage Score: 50.19 Best Advantage Score: 50.80 Best Epoch: 15\n",
      "==============================\n",
      "23epoch 64512/71695\n",
      " Learning Rate: 0.00040960 Learning Time: 7.563s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:22:58,113] [eval] Advantage Score: 50.93 Best Advantage Score: 50.80 Best Epoch: 15\n",
      "==============================\n",
      "24epoch 64512/71695\n",
      " Learning Rate: 0.00040960 Learning Time: 7.559s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:23:08,991] [eval] Advantage Score: 49.84 Best Advantage Score: 50.93 Best Epoch: 23\n",
      "==============================\n",
      "25epoch 64512/71695\n",
      " Learning Rate: 0.00040960 Learning Time: 7.522s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:23:19,692] [eval] Advantage Score: 51.09 Best Advantage Score: 50.93 Best Epoch: 23\n",
      "==============================\n",
      "26epoch 64512/71695\n",
      " Learning Rate: 0.00032768 Learning Time: 7.530s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:23:30,479] [eval] Advantage Score: 49.33 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "27epoch 64512/71695\n",
      " Learning Rate: 0.00032768 Learning Time: 7.622s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:23:41,234] [eval] Advantage Score: 49.89 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "28epoch 64512/71695\n",
      " Learning Rate: 0.00032768 Learning Time: 7.530s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:23:51,917] [eval] Advantage Score: 50.08 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "29epoch 64512/71695\n",
      " Learning Rate: 0.00032768 Learning Time: 7.540s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:24:02,544] [eval] Advantage Score: 46.66 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "30epoch 64512/71695\n",
      " Learning Rate: 0.00032768 Learning Time: 7.500s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:24:13,163] [eval] Advantage Score: 49.73 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "31epoch 64512/71695\n",
      " Learning Rate: 0.00026214 Learning Time: 7.517s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:24:23,798] [eval] Advantage Score: 50.02 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "32epoch 64512/71695\n",
      " Learning Rate: 0.00026214 Learning Time: 7.477s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:24:34,498] [eval] Advantage Score: 48.31 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "33epoch 64512/71695\n",
      " Learning Rate: 0.00026214 Learning Time: 7.519s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:24:45,210] [eval] Advantage Score: 47.91 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "34epoch 64512/71695\n",
      " Learning Rate: 0.00026214 Learning Time: 7.511s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:24:55,884] [eval] Advantage Score: 50.02 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "35epoch 64512/71695\n",
      " Learning Rate: 0.00026214 Learning Time: 7.525s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:25:06,566] [eval] Advantage Score: 50.27 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "36epoch 64512/71695\n",
      " Learning Rate: 0.00020972 Learning Time: 7.499s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:25:17,217] [eval] Advantage Score: 48.73 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "37epoch 64512/71695\n",
      " Learning Rate: 0.00020972 Learning Time: 7.541s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:25:27,868] [eval] Advantage Score: 50.21 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "38epoch 64512/71695\n",
      " Learning Rate: 0.00020972 Learning Time: 7.571s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:25:38,608] [eval] Advantage Score: 47.49 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "39epoch 64512/71695\n",
      " Learning Rate: 0.00020972 Learning Time: 7.566s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:25:49,357] [eval] Advantage Score: 50.59 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "40epoch 64512/71695\n",
      " Learning Rate: 0.00020972 Learning Time: 7.543s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:26:00,064] [eval] Advantage Score: 49.56 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "41epoch 64512/71695\n",
      " Learning Rate: 0.00016777 Learning Time: 7.493s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:26:10,731] [eval] Advantage Score: 50.21 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "42epoch 64512/71695\n",
      " Learning Rate: 0.00016777 Learning Time: 7.511s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:26:21,384] [eval] Advantage Score: 49.34 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "43epoch 64512/71695\n",
      " Learning Rate: 0.00016777 Learning Time: 7.521s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:26:31,997] [eval] Advantage Score: 49.71 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "44epoch 64512/71695\n",
      " Learning Rate: 0.00016777 Learning Time: 7.519s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:26:42,674] [eval] Advantage Score: 49.22 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "45epoch 64512/71695\n",
      " Learning Rate: 0.00016777 Learning Time: 7.535s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:26:53,333] [eval] Advantage Score: 48.31 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "46epoch 64512/71695\n",
      " Learning Rate: 0.00013422 Learning Time: 7.520s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:27:04,005] [eval] Advantage Score: 48.49 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "47epoch 64512/71695\n",
      " Learning Rate: 0.00013422 Learning Time: 7.527s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:27:14,658] [eval] Advantage Score: 50.23 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "48epoch 64512/71695\n",
      " Learning Rate: 0.00013422 Learning Time: 7.503s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:27:25,304] [eval] Advantage Score: 48.63 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "49epoch 64512/71695\n",
      " Learning Rate: 0.00013422 Learning Time: 7.537s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:27:35,999] [eval] Advantage Score: 48.12 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "==============================\n",
      "50epoch 64512/71695\n",
      " Learning Rate: 0.00013422 Learning Time: 7.523s\n",
      "crime train is completed\n",
      "priority train is completed\n",
      "Prior Advantage is Best\n",
      "crime eval is completed\n",
      "priority eval is completed\n",
      "[2021-07-21 12:27:46,647] [eval] Advantage Score: 49.41 Best Advantage Score: 51.09 Best Epoch: 25\n",
      "[2021-07-21 12:27:46,705] [best] [Mode train_mixed] [Best Crime Acc 74.15] [Best Crime F1 55.635]\n",
      "[2021-07-21 12:27:46,706] [best] [Mode train_mixed] [Best Priority Acc 66.44] [Best Priority F1 46.545]\n",
      "[2021-07-21 12:27:46,707] [best] [Best Advantage Score] 51.09024926810034\n",
      "==End==\n"
     ]
    }
   ],
   "source": [
    "# 데이터를 생성하고, 학습을 진행한 후에 모델 파라미터를 저장하는 Cell입니다.\n",
    "# 데이터 전처리 및 학습에 사용되는 Argument에 대한 Default value는 해당 노트북의 첫번째 Cell에 선언되어 있습니다.\n",
    "# 학습 결과로 만들어진 모델 파라미터가 노트북이 존재하는 경로에 생성된 training_data 폴더 내에 [학습이 진행된 시간] 폴더에 생성됩니다.\n",
    "\n",
    "configs['preprocess']=True\n",
    "learner=MixedLearner(logger, time_data, data_path, save_path, device, configs)\n",
    "learner.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 학습이 완료된 모델 파라미터를 이용하여 test.csv에 우범여부와 핵심적발에 대한 추론을 진행, test.csv에 추론 결과를 입력하여 저장하는 Class입니다.\n",
    "\n",
    "class RecordData:\n",
    "    def __init__(self,data_path,save_path,current_path,configs):\n",
    "        self.configs=configs\n",
    "        self.save_path=save_path\n",
    "        self.test_csv=copy.deepcopy(pd.read_csv(os.path.join(data_path,'test.csv')))\n",
    "        self.npy_dict=load_dataset(data_path,configs)\n",
    "        test_dataset=TensorDataset(torch.from_numpy(self.npy_dict['test_crime_data']).float(),torch.from_numpy(self.npy_dict['test_priority_data']).float())\n",
    "        self.data_loader=DataLoader(test_dataset)\n",
    "        self.configs=load_params(configs,current_path,configs['file_name'])\n",
    "        self.configs['file_name']=configs['file_name']\n",
    "        if 'crime' in configs['mode']:\n",
    "            self.input_space=self.data_loader.dataset[0][0].size()[0]\n",
    "            self.output_space=2\n",
    "        elif 'priority' in configs['mode']:\n",
    "            self.input_space=self.data_loader.dataset[0][0].size()[0]\n",
    "            self.output_space=2\n",
    "        else: #mixed\n",
    "            self.input_space=[self.data_loader.dataset[0][0].size()[0], self.data_loader.dataset[0][1].size()[0]]\n",
    "            self.output_space=2\n",
    "        self.model=MODEL[self.configs['mode'].split('_')[1]](self.input_space,self.output_space,configs)\n",
    "        self.metric={}\n",
    "        self.scoring_metric={}\n",
    "    \n",
    "    def _eval_before_save(self):\n",
    "        mixed_dataset=TensorDataset(torch.from_numpy(self.npy_dict['train_crime_data']).float(),torch.from_numpy(self.npy_dict['train_priority_data']).float(),torch.from_numpy(self.npy_dict['crime_targets']).long(),torch.from_numpy(self.npy_dict['priority_targets']).long())\n",
    "        # train_dataset=Subset(mixed_dataset,self.npy_dict['train_indices'])\n",
    "        valid_dataset=Subset(mixed_dataset,self.npy_dict['valid_indices'])\n",
    "        valid_dataloader=DataLoader(valid_dataset,batch_size=self.configs['batch_size'])\n",
    "        self.model.eval()\n",
    "        eval_loss=0.0\n",
    "        score_dict={\n",
    "            'loss':0.0,\n",
    "            'total':0.0,\n",
    "            'crime':{\n",
    "            'accuracy':0.0,\n",
    "            'total':0.0,\n",
    "            'precision':0.0,\n",
    "            'recall':0.0,\n",
    "            'f1score':0.0,\n",
    "            'loss':0.0,\n",
    "            'custom_loss':0.0,\n",
    "        },\n",
    "            'priority':{\n",
    "            'accuracy':0.0,\n",
    "            'total':0.0,\n",
    "            'precision':0.0,\n",
    "            'recall':0.0,\n",
    "            'f1score':0.0,\n",
    "            'loss':0.0,\n",
    "            'custom_loss':0.0,\n",
    "        },\n",
    "            'advantage':0.0\n",
    "        }\n",
    "        print(\"Start Eval\")\n",
    "        self.criterion=self.model.criterion\n",
    "        with torch.no_grad():\n",
    "            for batch_idx,(crime_data,priority_data,crime_targets,priority_targets) in enumerate(valid_dataloader):\n",
    "                crime_data,priority_data=crime_data.to(self.configs['device']),priority_data.to(self.configs['device'])\n",
    "                crime_targets,priority_targets=crime_targets.to(self.configs['device']),priority_targets.to(self.configs['device'])\n",
    "                \n",
    "                crime_outputs,priority_outputs=self.model(crime_data,priority_data)\n",
    "                crime_loss,priority_loss=self.criterion(crime_outputs,priority_outputs,crime_targets,priority_targets)\n",
    "                crime_predictions=torch.max(crime_outputs,dim=1)[1].clone()\n",
    "                priority_predictions=torch.max(priority_outputs,dim=1)[1].clone()\n",
    "\n",
    "                #loss\n",
    "                loss=crime_loss+priority_loss\n",
    "                score_dict['crime']['loss']+=crime_loss.item()\n",
    "                score_dict['priority']['loss']+=priority_loss.item()\n",
    "                priority_predictions[crime_predictions==0]=-1\n",
    "                self._save_scoring(crime_predictions,crime_targets,priority_predictions+1,priority_targets)\n",
    "                eval_loss +=loss.item()\n",
    "        # crime\n",
    "        score_dict['crime']=self._get_score(self.scoring_metric['crime']['predictions'],self.scoring_metric['crime']['targets'],score_dict['crime'])\n",
    "        # priority\n",
    "        score_dict['priority']=self._get_score(self.scoring_metric['priority']['predictions'],self.scoring_metric['priority']['targets'],score_dict['priority'])\n",
    "                \n",
    "        score_dict['loss']=eval_loss/(batch_idx+1)\n",
    "        score_dict['crime']['loss']=score_dict['crime']['loss']/(batch_idx+1)\n",
    "        score_dict['priority']['loss']=score_dict['priority']['loss']/(batch_idx+1)\n",
    "        score_dict['advantage'] = (score_dict['crime']['f1score'] + score_dict['priority']['f1score'])*0.5\n",
    "        print('[Crime Acc] {0:.4f}\\t [Priority Acc] {1:.4f}\\t'.format(score_dict['crime']['accuracy'], score_dict['priority']['accuracy']))\n",
    "        print('[Crime F1] {0:.4f}\\t [Priority F1] {1:.4f}\\t [Advantage] {2:.4f}\\t'.format(score_dict['crime']['f1score'], score_dict['priority']['f1score'], score_dict['advantage']))\n",
    "        print(\"End Eval\")\n",
    "\n",
    "    def run(self):\n",
    "        self._load_model()\n",
    "        print(\"Model Load Complete\")\n",
    "        self.model.to(self.configs['device'])\n",
    "        # eval하는거 성능측정 확인\n",
    "        #self._eval_before_save()\n",
    "\n",
    "        if 'mixed' not in self.configs['mode']:\n",
    "            metric=self.run_ind()\n",
    "        else: #mixed\n",
    "            metric=self.run_mix()\n",
    "        print(\"Prediction Complete\")\n",
    "        \n",
    "        self._record(metric)\n",
    "        print(\"Record Complete\")\n",
    "        \n",
    "    def run_mix(self):\n",
    "        self.model.eval()        \n",
    "        metric=dict()\n",
    "        with torch.no_grad():\n",
    "            for batch_idx,(crime_data,priority_data) in enumerate(self.data_loader):\n",
    "                crime_data=crime_data.to(self.configs['device'])\n",
    "                priority_data=priority_data.to(self.configs['device'])\n",
    "\n",
    "                crime_outputs,priority_outputs=self.model(crime_data,priority_data)\n",
    "                crime_predictions=torch.max(crime_outputs,dim=1)[1].clone()\n",
    "                priority_predictions=torch.max(priority_outputs,dim=1)[1].clone()\n",
    "\n",
    "                priority_predictions[crime_predictions==0]=-1\n",
    "                metric=self._save_score(crime_predictions,priority_predictions+1,metric)\n",
    "        return copy.deepcopy(metric)\n",
    "\n",
    "    def run_ind(self):\n",
    "        self.model.eval()\n",
    "\n",
    "        with torch.no_grad():\n",
    "            for batch_idx,data in enumerate(self.data_loader):\n",
    "                data=data.to(self.configs['device'])\n",
    "\n",
    "                outputs=self.model(data)\n",
    "                predictions=torch.max(outputs,dim=1)[1].clone() # cross-entropy\n",
    "                # predictions=torch.round(outputs).view(-1)#linear regression\n",
    "\n",
    "                if self.metric == dict():\n",
    "                    self.metric['predictions']=predictions\n",
    "                else:\n",
    "                    self.metric['predictions']=torch.cat((self.metric['predictions'],predictions),dim=0)\n",
    "        return copy.deepcopy(self.metric)\n",
    "\n",
    "    def _record(self,metric):\n",
    "        for key in self.test_csv.columns:\n",
    "            if key not in ['우범여부','핵심적발','신고번호']:\n",
    "                self.test_csv.drop(key,axis=1,inplace=True)\n",
    "        if self.configs['mode'].split('_')[1]=='mixed':\n",
    "            self.test_csv['우범여부']=metric['crime']['predictions'].cpu()\n",
    "            self.test_csv['핵심적발']=metric['priority']['predictions'].cpu()\n",
    "        elif self.configs['mode'].split('_')[1]=='crime':\n",
    "            self.test_csv['우범여부']=metric['predictions'].cpu()\n",
    "        elif self.configs['mode'].split('_')[1]=='priority':\n",
    "            self.test_csv['핵심적발']=metric['predictions'].cpu()\n",
    "        print('Checking length of test.csv :', len(self.test_csv.index))\n",
    "        self.test_csv.to_csv(os.path.join(self.save_path,self.configs['file_name']+'_test.csv'), index = False)\n",
    "        return\n",
    "        \n",
    "    def _load_model(self):\n",
    "        if 'mixed' not in self.configs['mode']:\n",
    "            dict_model=torch.load(os.path.join(self.save_path,self.configs['file_name'],'best_{}_model.pt').format(self.configs['mode'].split('_')[1]))\n",
    "            self.model.load_model(dict_model)\n",
    "        else: \n",
    "            crime_dict=copy.deepcopy(torch.load(os.path.join(self.save_path,self.configs['file_name'],'best_crime_model.pt')))\n",
    "            priority_dict=copy.deepcopy(torch.load(os.path.join(self.save_path,self.configs['file_name'],'best_priority_model.pt')))\n",
    "            # print(\"========== Performances ==========\")\n",
    "            # print(\"crime F1: {:.3f} crime Acc: {:.3f}\".format(crime_dict['f1score'],crime_dict['accuracy']))\n",
    "            # print(\"priority F1: {:.3f} priority Acc: {:.3f}\".format(priority_dict['f1score'],priority_dict['accuracy']))\n",
    "            # print(\"Advantage: {:.3f}\".format((crime_dict['f1score']+priority_dict['f1score'])*0.5))\n",
    "            # print(\"==================================\")\n",
    "            \n",
    "            # Modified\n",
    "            dict_model={**crime_dict,**priority_dict}\n",
    "            self.model.load_model(dict_model)\n",
    "\n",
    "            # Origin\n",
    "            # self.model.load_model(crime_dict,priority_dict)\n",
    "        \n",
    "\n",
    "    def save_models(self,epoch, score_dict):\n",
    "        dict_model={\n",
    "            'epoch':epoch,\n",
    "            '{}_model_state_dict'.format(self.configs['mode'].split('_')[1]):self.model.state_dict(),\n",
    "        }.update(score_dict)\n",
    "        torch.save(dict_model,\n",
    "        os.path.join(self.save_path,self.time_data,'best_model.pt'))\n",
    "    \n",
    "    def _save_score(self,crime_predictions,priority_predictions,metric):\n",
    "        if metric==dict():\n",
    "            metric['crime']=dict()\n",
    "            metric['priority']=dict()\n",
    "            metric['crime']['predictions']= crime_predictions\n",
    "            metric['priority']['predictions']= priority_predictions\n",
    "        else:\n",
    "            metric['crime']['predictions']=torch.cat((metric['crime']['predictions'],crime_predictions),dim=0)\n",
    "            metric['priority']['predictions']=torch.cat((metric['priority']['predictions'],priority_predictions),dim=0)\n",
    "\n",
    "        return metric\n",
    "    \n",
    "    def _save_scoring(self,crime_predictions,crime_targets,priority_predictions,priority_targets):\n",
    "        if self.scoring_metric==dict():\n",
    "            self.scoring_metric['crime']=dict()\n",
    "            self.scoring_metric['priority']=dict()\n",
    "            self.scoring_metric['crime']['predictions']= crime_predictions\n",
    "            self.scoring_metric['crime']['targets']= crime_targets\n",
    "            self.scoring_metric['priority']['predictions']= priority_predictions\n",
    "            self.scoring_metric['priority']['targets']= priority_targets\n",
    "        else:\n",
    "            self.scoring_metric['crime']['predictions']=torch.cat((self.scoring_metric['crime']['predictions'],crime_predictions),dim=0)\n",
    "            self.scoring_metric['crime']['targets']=torch.cat((self.scoring_metric['crime']['targets'],crime_targets),dim=0)\n",
    "            self.scoring_metric['priority']['predictions']=torch.cat((self.scoring_metric['priority']['predictions'],priority_predictions),dim=0)\n",
    "            self.scoring_metric['priority']['targets']=torch.cat((self.scoring_metric['priority']['targets'],priority_targets),dim=0)\n",
    "\n",
    "        return self.metric\n",
    "    \n",
    "    def _save_scoring(self,crime_predictions,crime_targets,priority_predictions,priority_targets):\n",
    "        if self.scoring_metric==dict():\n",
    "            self.scoring_metric['crime']=dict()\n",
    "            self.scoring_metric['priority']=dict()\n",
    "            self.scoring_metric['crime']['predictions']= crime_predictions\n",
    "            self.scoring_metric['crime']['targets']= crime_targets\n",
    "            self.scoring_metric['priority']['predictions']= priority_predictions\n",
    "            self.scoring_metric['priority']['targets']= priority_targets\n",
    "        else:\n",
    "            self.scoring_metric['crime']['predictions']=torch.cat((self.scoring_metric['crime']['predictions'],crime_predictions),dim=0)\n",
    "            self.scoring_metric['crime']['targets']=torch.cat((self.scoring_metric['crime']['targets'],crime_targets),dim=0)\n",
    "            self.scoring_metric['priority']['predictions']=torch.cat((self.scoring_metric['priority']['predictions'],priority_predictions),dim=0)\n",
    "            self.scoring_metric['priority']['targets']=torch.cat((self.scoring_metric['priority']['targets'],priority_targets),dim=0)\n",
    "\n",
    "    def _get_score(self,predictions,targets,score_dict):\n",
    "        from Utils.calc_score import calc_score\n",
    "        score_dict=calc_score(predictions,targets,score_dict)\n",
    "        return score_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Load Complete\n",
      "Prediction Complete\n",
      "Checking length of test.csv : 10273\n",
      "Record Complete\n"
     ]
    }
   ],
   "source": [
    "# 위의 Class를 실행하여, 학습된 모델 파라미터를 이용해 test.csv에 우범여부 및 핵심우범에 대한 추론 결과를 기록하여 저장하는 Cell입니다.\n",
    "# 해당 Cell의 실행 결과로 노트북이 존재하는 경로에 training_data 폴더 내에 [학습이 진행된 시간]_test.csv의 제출 결과물이 생성됩니다.\n",
    "\n",
    "configs['preprocess']=False\n",
    "configs['file_name']=time_data\n",
    "configs['mode']='record'\n",
    "runner=RecordData(data_path,save_path,current_path,configs)\n",
    "runner.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}