training_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.distributions import Categorical\n",
    "from torch.nn import init\n",
    "import copy\n",
    "import numpy as np\n",
    "\n",
    "import queue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpu = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RND(nn.Module):\n",
    "    def __init__(self,width = 240, height =256, channel = 3):\n",
    "        super(RND,self).__init__()\n",
    "        \n",
    "        self.width = width\n",
    "        self.height = height\n",
    "        self.channel = channel\n",
    "        \n",
    "        self.predictor = nn.Sequential(\n",
    "            nn.Conv2d(\n",
    "                in_channels=3,\n",
    "                out_channels=32,\n",
    "                kernel_size=8,\n",
    "                stride=4),\n",
    "            nn.LeakyReLU(),\n",
    "            nn.Conv2d(\n",
    "                in_channels=32,\n",
    "                out_channels=64,\n",
    "                kernel_size=4,\n",
    "                stride=2),\n",
    "            nn.LeakyReLU(),\n",
    "            nn.Conv2d(\n",
    "                in_channels=64,\n",
    "                out_channels=64,\n",
    "                kernel_size=3,\n",
    "                stride=1),\n",
    "            nn.LeakyReLU(),\n",
    "            Flatten(),\n",
    "            nn.Linear(46592, 512), # change\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, 512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, 512)\n",
    "            )\n",
    "        self.target = copy.deepcopy(self.predictor)\n",
    "        for p in self.modules():\n",
    "            if isinstance(p, nn.Conv2d):\n",
    "                init.orthogonal_(p.weight, np.sqrt(2))\n",
    "                p.bias.data.zero_()\n",
    "\n",
    "            if isinstance(p, nn.Linear):\n",
    "                init.orthogonal_(p.weight, np.sqrt(2))\n",
    "                p.bias.data.zero_()\n",
    "\n",
    "        for param in self.target.parameters():\n",
    "            param.requires_grad = False\n",
    "            \n",
    "    def forward(self, state):\n",
    "        target_feature = self.target(state)\n",
    "        predict_feature = self.predictor(state)\n",
    "        return predict_feature, target_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Flatten(nn.Module):\n",
    "    def forward(self, input):\n",
    "        return input.view(input.size(0), -1)\n",
    "class PPO(nn.Module):\n",
    "    def __init__(self,width = 240, height =256, channel = 3, action_dim = 7):\n",
    "        self.width = width\n",
    "        self.height = height\n",
    "        self.channel = channel\n",
    "        self.action_dim = action_dim\n",
    "        super(PPO,self).__init__()\n",
    "        self.memory = []\n",
    "        \n",
    "        self.basic = nn.Sequential(\\\n",
    "                    nn.Conv2d(in_channels = 3,\\\n",
    "                             out_channels = 32,\\\n",
    "                             kernel_size = 8,\\\n",
    "                              stride = 4),\n",
    "                    nn.ReLU(),\n",
    "                                   nn.Conv2d(in_channels = 32,\\\n",
    "                                           out_channels = 64,\\\n",
    "                                           kernel_size = 4,\\\n",
    "                                           stride = 2),\\\n",
    "                    nn.ReLU(),\\\n",
    "                    nn.Conv2d(in_channels = 64,\\\n",
    "                             out_channels = 64,\\\n",
    "                             kernel_size = 3,\\\n",
    "                             stride = 1),\n",
    "                                   nn.ReLU(),\\\n",
    "                                   Flatten(),\n",
    "                    nn.Linear(46592,256), #have to change       \n",
    "                    nn.ReLU(),\\\n",
    "                    nn.Linear(256,448),\n",
    "                    nn.ReLU()\n",
    "                    )\n",
    "        self.actor = nn.Sequential(\\\n",
    "                                  nn.Linear(448,448),\\\n",
    "                                  nn.ReLU(),\\\n",
    "                                  nn.Linear(448,self.action_dim)\\\n",
    "                                  )\n",
    "        \n",
    "        self.extrinsic_critic = nn.Linear(448,1)\n",
    "        self.intrinsic_critic = nn.Linear(448,1)\n",
    "        \n",
    "        init.orthogonal_(self.extrinsic_critic.weight, 0.01)\n",
    "        self.extrinsic_critic.bias.data.zero_()\n",
    "\n",
    "        init.orthogonal_(self.intrinsic_critic.weight, 0.01)\n",
    "        self.intrinsic_critic.bias.data.zero_()\n",
    "\n",
    "        for i in range(len(self.actor)):\n",
    "            if type(self.actor[i]) == nn.Linear:\n",
    "                init.orthogonal_(self.actor[i].weight, 0.01)\n",
    "                self.actor[i].bias.data.zero_()\n",
    "    def forward(self, x,dim = -1):\n",
    "        x = self.basic(x)\n",
    "        action = self.actor(x)\n",
    "        action_prob = F.softmax(action,dim = dim)\n",
    "        \n",
    "        intrinsic = self.intrinsic_critic(x)\n",
    "        extrinsic = self.extrinsic_critic(x)\n",
    "        return action_prob,extrinsic,intrinsic    \n",
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.0001\n",
    "gamma         = 0.98\n",
    "lmbda         = 0.95\n",
    "eps_clip      = 0.1\n",
    "K_epoch       = 4\n",
    "T_horizon     = 128\n",
    "critic_coef = 0.5\n",
    "ent_coef = 0.001\n",
    "intrinsic_gamma = 0.99\n",
    "extrinsic_gamma = 0.999\n",
    "update_proportion = 0.25\n",
    "\n",
    "extrinsic_advantage_coef = 2\n",
    "intrinsic_advantage_coef = 1\n",
    "class Agent(nn.Module):\n",
    "    def __init__(self,width=240,height=256,channel = 3,action_dim=7,learning_rate=0.0005):\n",
    "        self.width = width\n",
    "        self.height = height\n",
    "        self.channel = channel\n",
    "        self.action_dim = action_dim\n",
    "        \n",
    "        super(Agent,self).__init__()\n",
    "        \n",
    "        self.memory = []\n",
    "        self.intrinsic_queue = queue.Queue()\n",
    "        self.intrinsic_input_queue = queue.Queue()\n",
    "        self.ppo = PPO(self.width, self.height, self.channel, self.action_dim)\n",
    "        self.rnd = RND(self.width, self.height , self.channel)\n",
    "        self.optimizer = optim.Adam(self.parameters(),lr = learning_rate)\n",
    "    def put_data(self,data):\n",
    "        self.memory.append(data)\n",
    "    def make_batch(self):\n",
    "        state_list, action_list, extrinsic_reward_list, intrinsic_reward_list, next_state_list, \\\n",
    "        prob_list, extrinsic_done_list,intrinsic_done_list = [],[],[],[],[],[], [],[]\n",
    "        for data in self.memory:\n",
    "            state,action,extrinsic_reward, intrinsic_reward,next_state,prob,done = data\n",
    "            state_list.append(state)\n",
    "            action_list.append([action])\n",
    "            extrinsic_reward_list.append([extrinsic_reward])\n",
    "            intrinsic_reward_list.append([intrinsic_reward])\n",
    "\n",
    "            next_state_list.append(next_state)\n",
    "            extrinsic_done_mask = 0 if done else 1\n",
    "            extrinsic_done_list.append([extrinsic_done_mask])\n",
    "            intrinsic_done_list.append([1])\n",
    "            prob_list.append([prob])\n",
    "        self.memory = []\n",
    "\n",
    "        s,a,er,ir,next_s,extrinsic_done_list,intrinsic_done_list,prob \\\n",
    "                                        = torch.tensor(state_list,dtype=torch.float),\\\n",
    "                                        torch.tensor(action_list),\\\n",
    "                                        torch.tensor(extrinsic_reward_list,dtype = torch.float),\\\n",
    "                                        torch.tensor(intrinsic_reward_list,dtype = torch.float),\\\n",
    "                                        torch.tensor(next_state_list,dtype=torch.float),\\\n",
    "                                        torch.tensor(extrinsic_done_list,dtype = torch.float),\\\n",
    "                                        torch.tensor(intrinsic_done_list,dtype = torch.float),\\\n",
    "                                        torch.tensor(prob_list,dtype = torch.float)\n",
    "        if gpu:\n",
    "            return s.cuda(),a.cuda(),er.cuda(),ir.cuda(),next_s.cuda(),extrinsic_done_list.cuda()\\\n",
    "            ,intrinsic_done_list.cuda(),prob.cuda() \n",
    "        else :\n",
    "            return s,a,er,ir,next_s,extrinsic_done_list,intrinsic_done_list,prob  \n",
    "    \n",
    "    def train(self):\n",
    "        state,action,extrinsic_reward,intrinsic_reward, next_state,extrinsic_done_list,\\\n",
    "        intrinsic_done_list,action_prob = self.make_batch()\n",
    "        \n",
    "        for i in range(K_epoch):\n",
    "            state = state.squeeze()\n",
    "            next_state = next_state.squeeze()\n",
    "            predicted_action, predicted_extrinsic, predicted_intrinsic = self.ppo(state)\n",
    "            predicted_next_action, predicted_next_extrinsic, predicted_next_intrinsic = self.ppo(next_state)\n",
    "\n",
    "            if gpu:\n",
    "                intrinsic_next_state_mean = torch.mean(torch.cat(list(self.intrinsic_input_queue.queue)),dim = 0).cuda()\n",
    "                intrinsic_next_state_std = torch.std(torch.cat(list(model.intrinsic_input_queue.queue)),dim = 0).cuda()\n",
    "            \n",
    "                preprocessed_next_state = torch.clamp(((next_state - intrinsic_next_state_mean) / \\\n",
    "                                                   intrinsic_next_state_std + torch.tensor(1e-8).cuda()),-5,5)\n",
    "            else:\n",
    "                intrinsic_next_state_mean = torch.mean(torch.cat(list(self.intrinsic_input_queue.queue)),dim = 0)\n",
    "                intrinsic_next_state_std = torch.std(torch.cat(list(model.intrinsic_input_queue.queue)),dim = 0)\n",
    "            \n",
    "                preprocessed_next_state = torch.clamp(((next_state - intrinsic_next_state_mean) / \\\n",
    "                                                   intrinsic_next_state_std + torch.tensor(1e-8)),-5,5)\n",
    "            \n",
    "            predict_feature, target_feature = self.rnd(next_state)\n",
    "            td_error = extrinsic_reward + extrinsic_gamma * predicted_next_extrinsic * extrinsic_done_list\n",
    "            delta = td_error - predicted_next_extrinsic\n",
    "            if gpu:\n",
    "                delta = delta.detach().cpu().numpy()\n",
    "            else:\n",
    "                delta = delta.detach().numpy()\n",
    "            advantage_list = []\n",
    "            \n",
    "            advantage = 0.0\n",
    "            for delta_t in delta[::-1]:\n",
    "                advantage = gamma * lmbda * advantage + delta_t[0]\n",
    "                advantage_list.append([advantage])\n",
    "            advantage_list.reverse()\n",
    "            if gpu:\n",
    "                advantage = torch.tensor(advantage_list,dtype = torch.float).cuda()\n",
    "            else:\n",
    "                advantage = torch.tensor(advantage_list,dtype = torch.float)\n",
    "            ##intrinsic_advantage\n",
    "            intrinsic_td_error = intrinsic_reward + intrinsic_gamma * predicted_next_intrinsic * intrinsic_done_list\n",
    "            intrinsic_delta = intrinsic_td_error - predicted_next_intrinsic\n",
    "            if gpu:\n",
    "                intrinsic_delta = intrinsic_delta.detach().cpu().numpy()\n",
    "            else:\n",
    "                intrinsic_delta = intrinsic_delta.detach().numpy()\n",
    "                \n",
    "            intrinsic_advantage_list = []\n",
    "            intrinsic_advantage = 0.0\n",
    "            \n",
    "            for intrinsic_delta_t in intrinsic_delta[::-1]:\n",
    "                intrinsic_advantage = gamma * lmbda * intrinsic_advantage + intrinsic_delta_t[0]\n",
    "                intrinsic_advantage_list.append([intrinsic_advantage])\n",
    "            intrinsic_advantage_list.reverse()\n",
    "            if gpu:\n",
    "                intrinsic_advantage = torch.tensor(intrinsic_advantage_list,dtype = torch.float).cuda()\n",
    "            else:\n",
    "                intrinsic_advantage = torch.tensor(intrinsic_advantage_list,dtype = torch.float)\n",
    "            #### intrinsic_error\n",
    "            intrinsic_error = (intrinsic_td_error - predicted_intrinsic.detach()).pow(2)\n",
    "            if gpu:\n",
    "                masking = torch.rand(len(intrinsic_error)).cuda()\n",
    "                masking = (masking < update_proportion).type(torch.FloatTensor).cuda()\n",
    "            else:\n",
    "                masking = torch.rand(len(intrinsic_error))\n",
    "                masking = (masking < update_proportion).type(torch.FloatTensor)\n",
    "            if gpu:\n",
    "                intrinsic_error = (intrinsic_error * masking).sum() / torch.max(intrinsic_error.sum(), torch.Tensor([1]).cuda())\n",
    "            else:\n",
    "                intrinsic_error = (intrinsic_error * masking).sum() / torch.max(masking.sum(), torch.Tensor([1]))\n",
    "\n",
    "            \n",
    "            now_action = predicted_action\n",
    "            m = Categorical(now_action)\n",
    "            entropy = m.entropy().mean()\n",
    "            \n",
    "            \n",
    "            now_action = now_action.gather(1,action)\n",
    "            \n",
    "            \n",
    "            ratio = torch.exp(torch.log(now_action) - torch.log(action_prob))\n",
    "            advantage = extrinsic_advantage_coef * advantage +  intrinsic_advantage_coef * intrinsic_advantage\n",
    "            surr1 = ratio * advantage\n",
    "            surr2 = torch.clamp(ratio , 1-eps_clip, 1 + eps_clip) * advantage\n",
    "            loss = - torch.min(surr1,surr2) + critic_coef * (F.smooth_l1_loss(predicted_extrinsic,td_error.detach()) +\\\n",
    "                    intrinsic_error) - ent_coef * entropy + F.mse_loss(predict_feature, target_feature)\n",
    "            \n",
    "            self.optimizer.zero_grad()\n",
    "            loss.mean().backward()\n",
    "            self.optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 1000\n",
    "from nes_py.wrappers import JoypadSpace\n",
    "import gym_super_mario_bros\n",
    "from gym_super_mario_bros.actions import SIMPLE_MOVEMENT\n",
    "env = gym_super_mario_bros.make('SuperMarioBros-v0')\n",
    "env = JoypadSpace(env, SIMPLE_MOVEMENT)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "if gpu:\n",
    "    model = Agent().cuda()\n",
    "else:\n",
    "    model = Agent()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "try:\n",
    "    if gpu:\n",
    "        model.load_state_dict(torch.load(\"weights/2_episode_model\"))\n",
    "    else:\n",
    "        model.load_state_dict(torch.load(\"weights/2_episode_model\", map_location={'cuda:0': 'cpu'}))\n",
    "except :\n",
    "    pass\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "intrinsic_reward.item() :  0.0\n",
      "global_step :  1 , action :  3 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285729825496674, 0.14285695552825928]\n",
      "extrinsic_reward :  0 intrinsic_reward :  0.0\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "intrinsic_reward.item() :  0.0\n",
      "global_step :  2 , action :  2 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285729825496674, 0.14285695552825928]\n",
      "extrinsic_reward :  0 intrinsic_reward :  0.0\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "intrinsic_reward.item() :  1.9368853569030762\n",
      "global_step :  3 , action :  1 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285729825496674, 0.14285695552825928]\n",
      "extrinsic_reward :  0 intrinsic_reward :  1.4142135381698608\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 84}\n",
      "intrinsic_reward.item() :  5.147770404815674\n",
      "global_step :  4 , action :  2 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285728335380554, 0.14285694062709808]\n",
      "extrinsic_reward :  0 intrinsic_reward :  1.605045199394226\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 89}\n",
      "intrinsic_reward.item() :  3729.90087890625\n",
      "global_step :  5 , action :  0 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285729825496674, 0.14285695552825928]\n",
      "extrinsic_reward :  0 intrinsic_reward :  1.9999985694885254\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 93}\n",
      "intrinsic_reward.item() :  9.523905754089355\n",
      "global_step :  6 , action :  4 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285726845264435, 0.14285728335380554, 0.14285695552825928]\n",
      "extrinsic_reward :  1 intrinsic_reward :  -0.44274649024009705\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 96}\n",
      "intrinsic_reward.item() :  8450.8818359375\n",
      "global_step :  7 , action :  5 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285728335380554, 0.14285694062709808]\n",
      "extrinsic_reward :  0 intrinsic_reward :  2.2172791957855225\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 99}\n",
      "intrinsic_reward.item() :  8438.34375\n",
      "global_step :  8 , action :  5 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285728335380554, 0.14285694062709808]\n",
      "extrinsic_reward :  0 intrinsic_reward :  1.6304080486297607\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 101}\n",
      "intrinsic_reward.item() :  8399.0517578125\n",
      "global_step :  9 , action :  2 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285729825496674, 0.14285695552825928]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.3435860872268677\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 102}\n",
      "intrinsic_reward.item() :  13598.7919921875\n",
      "global_step :  10 , action :  2 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285729825496674, 0.14285695552825928]\n",
      "extrinsic_reward :  0 intrinsic_reward :  1.9455392360687256\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 103}\n",
      "intrinsic_reward.item() :  8835.162109375\n",
      "global_step :  11 , action :  2 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285728335380554, 0.14285695552825928]\n",
      "extrinsic_reward :  1 intrinsic_reward :  0.8731271624565125\n",
      "place 43\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 43, 'x_pos_screen': 43, 'y_pos': 103}\n",
      "intrinsic_reward.item() :  8843.7607421875\n",
      "global_step :  12 , action :  5 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285729825496674, 0.14285695552825928]\n",
      "extrinsic_reward :  0 intrinsic_reward :  0.8121801018714905\n",
      "place 43\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 43, 'x_pos_screen': 43, 'y_pos': 102}\n",
      "intrinsic_reward.item() :  9383.767578125\n",
      "global_step :  13 , action :  2 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285728335380554, 0.14285695552825928]\n",
      "extrinsic_reward :  1 intrinsic_reward :  0.8626329898834229\n",
      "place 44\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 44, 'x_pos_screen': 44, 'y_pos': 101}\n",
      "intrinsic_reward.item() :  9452.55859375\n",
      "global_step :  14 , action :  2 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285728335380554, 0.14285695552825928]\n",
      "extrinsic_reward :  1 intrinsic_reward :  0.8231418132781982\n",
      "place 45\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 45, 'x_pos_screen': 45, 'y_pos': 99}\n",
      "intrinsic_reward.item() :  10068.2939453125\n",
      "global_step :  15 , action :  1 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285728335380554, 0.14285694062709808]\n",
      "extrinsic_reward :  0 intrinsic_reward :  0.8971595764160156\n",
      "place 45\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 45, 'x_pos_screen': 45, 'y_pos': 96}\n",
      "intrinsic_reward.item() :  10088.9306640625\n",
      "global_step :  16 , action :  3 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285729825496674, 0.14285694062709808]\n",
      "extrinsic_reward :  1 intrinsic_reward :  0.8516529202461243\n",
      "place 46\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 46, 'x_pos_screen': 46, 'y_pos': 93}\n",
      "intrinsic_reward.item() :  9996.6865234375\n",
      "global_step :  17 , action :  0 action_prob :  [0.14285679161548615, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285729825496674, 0.14285695552825928]\n",
      "extrinsic_reward :  1 intrinsic_reward :  0.7906783819198608\n",
      "place 47\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 47, 'x_pos_screen': 47, 'y_pos': 89}\n",
      "intrinsic_reward.item() :  22942.44140625\n",
      "global_step :  18 , action :  6 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285708963871002, 0.14285728335380554, 0.14285729825496674, 0.14285695552825928]\n",
      "extrinsic_reward :  1 intrinsic_reward :  2.6933021545410156\n",
      "place 48\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 400, 'world': 1, 'x_pos': 48, 'x_pos_screen': 48, 'y_pos': 85}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "intrinsic_reward.item() :  22939.35546875\n",
      "global_step :  19 , action :  0 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285729825496674, 0.14285694062709808]\n",
      "extrinsic_reward :  -1 intrinsic_reward :  2.2297821044921875\n",
      "place 48\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 48, 'x_pos_screen': 48, 'y_pos': 79}\n",
      "intrinsic_reward.item() :  31892.9140625\n",
      "global_step :  20 , action :  3 action_prob :  [0.14285677671432495, 0.14285710453987122, 0.14285744726657867, 0.14285707473754883, 0.14285728335380554, 0.14285729825496674, 0.14285695552825928]\n",
      "extrinsic_reward :  1 intrinsic_reward :  2.7246546745300293\n",
      "place 49\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 49, 'x_pos_screen': 49, 'y_pos': 79}\n",
      "intrinsic_reward.item() :  37568.2734375\n",
      "global_step :  21 , action :  5 action_prob :  [0.12556615471839905, 0.12546494603157043, 0.19100242853164673, 0.1250806599855423, 0.12483567744493484, 0.18413619697093964, 0.1239139512181282]\n",
      "extrinsic_reward :  1 intrinsic_reward :  2.6701338291168213\n",
      "place 50\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 50, 'x_pos_screen': 50, 'y_pos': 79}\n",
      "intrinsic_reward.item() :  43134.7890625\n",
      "global_step :  22 , action :  2 action_prob :  [0.12555880844593048, 0.1254575401544571, 0.19102294743061066, 0.1250731348991394, 0.12482807785272598, 0.18415327370166779, 0.12390604615211487]\n",
      "extrinsic_reward :  1 intrinsic_reward :  2.595402479171753\n",
      "place 51\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 51, 'x_pos_screen': 51, 'y_pos': 84}\n",
      "intrinsic_reward.item() :  43282.83984375\n",
      "global_step :  23 , action :  3 action_prob :  [0.12556330859661102, 0.12546207010746002, 0.19101044535636902, 0.1250777393579483, 0.12483272701501846, 0.18414287269115448, 0.12391088902950287]\n",
      "extrinsic_reward :  0 intrinsic_reward :  2.2406530380249023\n",
      "place 51\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 51, 'x_pos_screen': 51, 'y_pos': 89}\n",
      "intrinsic_reward.item() :  43386.66796875\n",
      "global_step :  24 , action :  4 action_prob :  [0.1255692094564438, 0.12546798586845398, 0.19099390506744385, 0.12508375942707062, 0.12483882904052734, 0.18412908911705017, 0.12391722202301025]\n",
      "extrinsic_reward :  1 intrinsic_reward :  2.000488519668579\n",
      "place 52\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 52, 'x_pos_screen': 52, 'y_pos': 93}\n",
      "intrinsic_reward.item() :  43749.2265625\n",
      "global_step :  25 , action :  2 action_prob :  [0.12557560205459595, 0.12547442317008972, 0.19097602367401123, 0.12509030103683472, 0.12484543025493622, 0.1841142177581787, 0.12392410635948181]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.8396401405334473\n",
      "place 53\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 53, 'x_pos_screen': 53, 'y_pos': 97}\n",
      "intrinsic_reward.item() :  43320.25390625\n",
      "global_step :  26 , action :  6 action_prob :  [0.12558531761169434, 0.12548419833183289, 0.1909487247467041, 0.1251002550125122, 0.12485548853874207, 0.1840914785861969, 0.12393457442522049]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.6736456155776978\n",
      "place 54\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 54, 'x_pos_screen': 54, 'y_pos': 100}\n",
      "intrinsic_reward.item() :  43269.0078125\n",
      "global_step :  27 , action :  3 action_prob :  [0.12558941543102264, 0.1254883110523224, 0.19093717634677887, 0.12510444223880768, 0.12485972046852112, 0.18408185243606567, 0.12393898516893387]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.560513973236084\n",
      "place 55\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 55, 'x_pos_screen': 55, 'y_pos': 103}\n",
      "intrinsic_reward.item() :  43283.8203125\n",
      "global_step :  28 , action :  1 action_prob :  [0.12559889256954193, 0.12549781799316406, 0.1909106820821762, 0.12511411309242249, 0.12486951053142548, 0.1840597689151764, 0.12394917756319046]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.470602035522461\n",
      "place 56\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 56, 'x_pos_screen': 56, 'y_pos': 104}\n",
      "intrinsic_reward.item() :  43321.19140625\n",
      "global_step :  29 , action :  5 action_prob :  [0.12560465931892395, 0.12550361454486847, 0.19089452922344208, 0.12512001395225525, 0.12487546354532242, 0.18404631316661835, 0.1239553838968277]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.3959654569625854\n",
      "place 57\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 57, 'x_pos_screen': 57, 'y_pos': 106}\n",
      "intrinsic_reward.item() :  43347.921875\n",
      "global_step :  30 , action :  1 action_prob :  [0.12559902667999268, 0.1254979521036148, 0.19091032445430756, 0.12511426210403442, 0.12486965209245682, 0.18405947089195251, 0.1239493191242218]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.3313990831375122\n",
      "place 58\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 58, 'x_pos_screen': 58, 'y_pos': 106}\n",
      "intrinsic_reward.item() :  43144.765625\n",
      "global_step :  31 , action :  6 action_prob :  [0.1255999058485031, 0.12549884617328644, 0.19090785086154938, 0.12511517107486725, 0.12487056106328964, 0.18405741453170776, 0.12395027279853821]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.2631176710128784\n",
      "place 59\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 59, 'x_pos_screen': 59, 'y_pos': 106}\n",
      "intrinsic_reward.item() :  43068.8125\n",
      "global_step :  32 , action :  6 action_prob :  [0.12559905648231506, 0.1254979968070984, 0.19091029465198517, 0.1251142919063568, 0.1248696893453598, 0.18405945599079132, 0.12394935637712479]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.20933997631073\n",
      "place 60\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 60, 'x_pos_screen': 60, 'y_pos': 105}\n",
      "intrinsic_reward.item() :  43023.01953125\n",
      "global_step :  33 , action :  0 action_prob :  [0.12559625506401062, 0.12549516558647156, 0.19091807305812836, 0.1251114308834076, 0.1248667761683464, 0.18406593799591064, 0.12394633889198303]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.1629180908203125\n",
      "place 61\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 61, 'x_pos_screen': 61, 'y_pos': 104}\n",
      "intrinsic_reward.item() :  42569.7265625\n",
      "global_step :  34 , action :  6 action_prob :  [0.12559874355793, 0.12549765408039093, 0.19091114401817322, 0.12511396408081055, 0.12486934661865234, 0.18406015634536743, 0.12394900619983673]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.0996119976043701\n",
      "place 62\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 62, 'x_pos_screen': 62, 'y_pos': 101}\n",
      "intrinsic_reward.item() :  42637.8125\n",
      "global_step :  35 , action :  3 action_prob :  [0.12560267746448517, 0.1255016177892685, 0.19090010225772858, 0.12511798739433289, 0.12487341463565826, 0.18405096232891083, 0.12395323812961578]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.0691474676132202\n",
      "place 63\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 63, 'x_pos_screen': 63, 'y_pos': 99}\n",
      "intrinsic_reward.item() :  42773.39453125\n",
      "global_step :  36 , action :  3 action_prob :  [0.12559561431407928, 0.12549450993537903, 0.1909198760986328, 0.12511076033115387, 0.12486610561609268, 0.18406742811203003, 0.12394563108682632]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.0450413227081299\n",
      "place 64\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 64, 'x_pos_screen': 64, 'y_pos': 95}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "intrinsic_reward.item() :  42682.17578125\n",
      "global_step :  37 , action :  0 action_prob :  [0.12560036778450012, 0.12549930810928345, 0.1909065693616867, 0.12511563301086426, 0.12487104535102844, 0.184056356549263, 0.1239507645368576]\n",
      "extrinsic_reward :  1 intrinsic_reward :  1.0110849142074585\n",
      "place 65\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 65, 'x_pos_screen': 65, 'y_pos': 91}\n",
      "intrinsic_reward.item() :  15310.85546875\n",
      "global_step :  38 , action :  6 action_prob :  [0.12560328841209412, 0.12550222873687744, 0.19089840352535248, 0.12511862814426422, 0.1248740553855896, 0.1840495616197586, 0.12395390123128891]\n",
      "extrinsic_reward :  1 intrinsic_reward :  -0.5207064747810364\n",
      "place 66\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 399, 'world': 1, 'x_pos': 66, 'x_pos_screen': 66, 'y_pos': 86}\n",
      "intrinsic_reward.item() :  15174.3798828125\n",
      "global_step :  39 , action :  0 action_prob :  [0.12560853362083435, 0.12550751864910126, 0.19088366627693176, 0.1251239776611328, 0.12487947940826416, 0.18403726816177368, 0.12395957112312317]\n",
      "extrinsic_reward :  0 intrinsic_reward :  -0.5197862386703491\n",
      "place 67\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 398, 'world': 1, 'x_pos': 67, 'x_pos_screen': 67, 'y_pos': 81}\n",
      "intrinsic_reward.item() :  14834.9833984375\n",
      "global_step :  40 , action :  1 action_prob :  [0.12560199201107025, 0.12550094723701477, 0.1909020096063614, 0.12511730194091797, 0.12487272918224335, 0.18405255675315857, 0.12395252287387848]\n",
      "extrinsic_reward :  1 intrinsic_reward :  -0.5305503606796265\n",
      "place 68\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 1, 'status': 'small', 'time': 398, 'world': 1, 'x_pos': 68, 'x_pos_screen': 68, 'y_pos': 79}\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-40-c9cae4eae3a6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     74\u001b[0m                 \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     75\u001b[0m             \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 76\u001b[1;33m         \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     77\u001b[0m     \u001b[1;31m#env.render()\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     78\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-5-1773beef29b3>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    145\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    146\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 147\u001b[1;33m             \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    148\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\unity\\lib\\site-packages\\torch\\tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[0;32m    100\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[1;33m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m         \"\"\"\n\u001b[1;32m--> 102\u001b[1;33m         \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    103\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    104\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\unity\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[0;32m     88\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[0;32m     89\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 90\u001b[1;33m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[0;32m     91\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     92\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "T_horizon = 20\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    global_step = 0\n",
    "    model.intrinsic_queue = queue.Queue()\n",
    "    model.intrinsic_input_queue = queue.Queue()\n",
    "    state = env.reset()\n",
    "    state = np.array(state)/255\n",
    "            #state = np.transpose(state,(2,0,1))\n",
    "    state = np.moveaxis(state, -1, 0)\n",
    "    state = torch.tensor(state).float()\n",
    "    state = state.unsqueeze(0)\n",
    "    done = False\n",
    "    while not done :\n",
    "        for t in range(T_horizon):\n",
    "            #env.render()\n",
    "            global_step +=1\n",
    "\n",
    "            if gpu:\n",
    "                action_prob, _ , _ = model.ppo.forward(state.cuda())\n",
    "            else:\n",
    "                action_prob, _ , _ = model.ppo.forward(state)\n",
    "            m = Categorical(action_prob)\n",
    "            action = m.sample().item()\n",
    "            \n",
    "            \n",
    "            next_state, extrinsic_reward, done, info = env.step(action)\n",
    "            next_state = np.array(next_state)/255\n",
    "            next_state = np.moveaxis(next_state,-1,0)\n",
    "            next_state = torch.tensor(next_state).float()\n",
    "            next_state = next_state.unsqueeze(0)\n",
    "            \n",
    "            \n",
    "            model.intrinsic_input_queue.put(next_state)\n",
    "            if len(model.intrinsic_input_queue.queue) > 128:\n",
    "                model.intrinsic_input_queue.get()\n",
    "            intrinsic_next_state_mean = \\\n",
    "                    torch.mean(torch.cat(list(model.intrinsic_input_queue.queue)),dim = 0)\n",
    "            if len(model.intrinsic_input_queue.queue) == 1:\n",
    "                intrinsic_next_state_std = torch.zeros(1)\n",
    "            else:\n",
    "                intrinsic_next_state_std = \\\n",
    "                        torch.std(torch.cat(list(model.intrinsic_input_queue.queue)),dim = 0)\n",
    "    \n",
    "            preprocessed_next_state = \\\n",
    "                    torch.clamp(((next_state - intrinsic_next_state_mean) / (intrinsic_next_state_std + 1e-8)), -5,5)\n",
    "            \n",
    "            \n",
    "            #(model.intrinsic_input_queue)\n",
    "            if gpu:\n",
    "                predictor,target = model.rnd.forward(preprocessed_next_state.cuda())\n",
    "            else:\n",
    "                predictor,target = model.rnd.forward(preprocessed_next_state)\n",
    "            intrinsic_reward = (predictor - target).pow(2).sum(1) / 2\n",
    "            if len(model.intrinsic_queue.queue) > 128:\n",
    "                model.intrinsic_queue.get()\n",
    "            model.intrinsic_queue.put(intrinsic_reward.item())\n",
    "            intrinsic_mean = np.mean(model.intrinsic_queue.queue)\n",
    "            intrinsic_std = np.std(model.intrinsic_queue.queue)\n",
    "            intrinsic_reward = (intrinsic_reward - intrinsic_mean) / (intrinsic_std+ 1e-8)\n",
    "            \n",
    "            if info['time'] == 0 or info['life'] == 1 or info['time'] == 300:\n",
    "                done = True\n",
    "                reward = -10.\n",
    "\n",
    "            model.put_data((state.tolist(), action, extrinsic_reward/100, (intrinsic_reward.item()), next_state.tolist(), action_prob[0][action].item(), done))\n",
    "            print('global_step : ',global_step,', action : ', action,'action_prob : ',action_prob.tolist()[0])\n",
    "            print('extrinsic_reward : ',extrinsic_reward,'intrinsic_reward : ',intrinsic_reward.item())\n",
    "            print('place',info['x_pos'])\n",
    "            print('info',info)\n",
    "            if done :\n",
    "                print('epoch : ',epoch, ', global_step : ',global_step)\n",
    "                break\n",
    "            state = next_state\n",
    "        model.train()\n",
    "    #env.render()\n",
    "\n",
    "#env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "deque([nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan,\n",
       "       nan])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.intrinsic_queue.queue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[[-5., -5., -5.,  ..., -5., -5., -5.],\n",
       "          [-5., -5., -5.,  ..., -5., -5., -5.],\n",
       "          [-5., -5., -5.,  ..., -5., -5., -5.],\n",
       "          ...,\n",
       "          [-5.,  5.,  5.,  ...,  5.,  5., nan],\n",
       "          [-5.,  5.,  5.,  ...,  5., nan, nan],\n",
       "          [ 5., nan, nan,  ..., nan, nan,  5.]],\n",
       "\n",
       "         [[-5., -5., -5.,  ..., -5., -5., -5.],\n",
       "          [-5., -5., -5.,  ..., -5., -5., -5.],\n",
       "          [-5., -5., -5.,  ..., -5., -5., -5.],\n",
       "          ...,\n",
       "          [-5., -5., -5.,  ..., -5., -5., nan],\n",
       "          [-5., -5., -5.,  ..., -5., nan, nan],\n",
       "          [-5., nan, nan,  ..., nan, nan, -5.]],\n",
       "\n",
       "         [[-5., -5., -5.,  ..., -5., -5., -5.],\n",
       "          [-5., -5., -5.,  ..., -5., -5., -5.],\n",
       "          [-5., -5., -5.,  ..., -5., -5., -5.],\n",
       "          ...,\n",
       "          [nan, nan, nan,  ..., nan, nan, nan],\n",
       "          [nan, nan, nan,  ..., nan, nan, nan],\n",
       "          [nan, nan, nan,  ..., nan, nan, nan]]]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessed_next_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "319"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "env has already been closed.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-44-1baceacf4cb1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\unity\\lib\\site-packages\\gym\\core.py\u001b[0m in \u001b[0;36mclose\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    234\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    235\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 236\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    237\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    238\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mseed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\unity\\lib\\site-packages\\gym\\core.py\u001b[0m in \u001b[0;36mclose\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    234\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    235\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 236\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    237\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    238\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mseed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\unity\\lib\\site-packages\\nes_py\\nes_env.py\u001b[0m in \u001b[0;36mclose\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    336\u001b[0m         \u001b[1;31m# make sure the environment hasn't already been closed\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    337\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_env\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 338\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'env has already been closed.'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    339\u001b[0m         \u001b[1;31m# purge the environment from C++ memory\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    340\u001b[0m         \u001b[0m_LIB\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mClose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_env\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: env has already been closed."
     ]
    }
   ],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "if gpu:\n",
    "    model.load_state_dict(torch.load(\"weights/3\"))\n",
    "else:\n",
    "    model.load_state_dict(torch.load(\"weights/3\", map_location={'cuda:0': 'cpu'}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 1000\n",
    "from nes_py.wrappers import JoypadSpace\n",
    "import gym_super_mario_bros\n",
    "from gym_super_mario_bros.actions import SIMPLE_MOVEMENT\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym_super_mario_bros.make('SuperMarioBros-1-3-v0')\n",
    "env = JoypadSpace(env, SIMPLE_MOVEMENT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  1 , action :  4 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.56402587890625\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  2 , action :  2 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.56402587890625\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 84}\n",
      "global_step :  3 , action :  1 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  413.3307800292969\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 89}\n",
      "global_step :  4 , action :  0 action_prob :  [0.05978989228606224, 0.1765068620443344, 0.04407934099435806, 0.5568422079086304, 0.07250624150037766, 0.04286649823188782, 0.04740886762738228]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.8639221191406\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 93}\n",
      "global_step :  5 , action :  4 action_prob :  [0.05976428836584091, 0.17649048566818237, 0.044056519865989685, 0.5569798350334167, 0.0724792331457138, 0.04284464567899704, 0.04738491773605347]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.188232421875\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 97}\n",
      "global_step :  6 , action :  4 action_prob :  [0.05968368053436279, 0.17643778026103973, 0.04398546367883682, 0.5574111938476562, 0.07239601016044617, 0.04277447983622551, 0.047311339527368546]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.49737548828125\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 100}\n",
      "global_step :  7 , action :  1 action_prob :  [0.059704381972551346, 0.1764526069164276, 0.0440034382045269, 0.5572999119758606, 0.07241684198379517, 0.042793162167072296, 0.0473296232521534]\n",
      "extrinsic_reward :  0 intrinsic_reward :  414.0\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 103}\n",
      "global_step :  8 , action :  0 action_prob :  [0.05966486409306526, 0.17642614245414734, 0.04396839812397957, 0.5575129389762878, 0.07237570732831955, 0.04275871068239212, 0.04729333147406578]\n",
      "extrinsic_reward :  0 intrinsic_reward :  413.5565185546875\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 104}\n",
      "global_step :  9 , action :  1 action_prob :  [0.059683870524168015, 0.17643922567367554, 0.043984923511743546, 0.557411253452301, 0.07239478081464767, 0.04277579486370087, 0.04731012508273125]\n",
      "extrinsic_reward :  1 intrinsic_reward :  413.386474609375\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 106}\n",
      "global_step :  10 , action :  5 action_prob :  [0.059737809002399445, 0.1764756143093109, 0.044032447040081024, 0.5571213364601135, 0.0724504292011261, 0.04282309114933014, 0.04735922813415527]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.66156005859375\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 106}\n",
      "global_step :  11 , action :  4 action_prob :  [0.059727638959884644, 0.17646794021129608, 0.04402357339859009, 0.5571768283843994, 0.07243993133306503, 0.04281395301222801, 0.04735007882118225]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.66156005859375\n",
      "place 43\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 43, 'x_pos_screen': 43, 'y_pos': 106}\n",
      "global_step :  12 , action :  3 action_prob :  [0.059727638959884644, 0.17646794021129608, 0.04402357339859009, 0.5571768283843994, 0.07243993133306503, 0.04281395301222801, 0.04735007882118225]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.006591796875\n",
      "place 43\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 43, 'x_pos_screen': 43, 'y_pos': 105}\n",
      "global_step :  13 , action :  2 action_prob :  [0.05978003144264221, 0.17650336027145386, 0.04406954348087311, 0.556895911693573, 0.07249358296394348, 0.04286021739244461, 0.047397371381521225]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.0084533691406\n",
      "place 44\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 44, 'x_pos_screen': 44, 'y_pos': 104}\n",
      "global_step :  14 , action :  3 action_prob :  [0.05974931642413139, 0.1764824539422989, 0.044042520225048065, 0.5570610165596008, 0.07246199995279312, 0.042833130806684494, 0.04736952483654022]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.8680419921875\n",
      "place 45\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 45, 'x_pos_screen': 45, 'y_pos': 101}\n",
      "global_step :  15 , action :  4 action_prob :  [0.0597805492579937, 0.1765044927597046, 0.0440698079764843, 0.5568927526473999, 0.07249385863542557, 0.04286102578043938, 0.04739750921726227]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.42626953125\n",
      "place 46\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 46, 'x_pos_screen': 46, 'y_pos': 99}\n",
      "global_step :  16 , action :  3 action_prob :  [0.05974053591489792, 0.17647723853588104, 0.044034600257873535, 0.5571079850196838, 0.07245275378227234, 0.04282567277550697, 0.04736127704381943]\n",
      "extrinsic_reward :  0 intrinsic_reward :  410.6059265136719\n",
      "place 46\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 46, 'x_pos_screen': 46, 'y_pos': 95}\n",
      "global_step :  17 , action :  3 action_prob :  [0.059752631932497025, 0.1764848679304123, 0.04404561594128609, 0.5570423603057861, 0.07246582210063934, 0.0428357794880867, 0.04737294092774391]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.9836730957031\n",
      "place 47\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 47, 'x_pos_screen': 47, 'y_pos': 91}\n",
      "global_step :  18 , action :  1 action_prob :  [0.05978779494762421, 0.17650938034057617, 0.04407648369669914, 0.5568526983261108, 0.07250197231769562, 0.0428670197725296, 0.04740467295050621]\n",
      "extrinsic_reward :  1 intrinsic_reward :  408.5539245605469\n",
      "place 48\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 48, 'x_pos_screen': 48, 'y_pos': 86}\n",
      "global_step :  19 , action :  3 action_prob :  [0.05977977439761162, 0.17650333046913147, 0.04406964033842087, 0.556895911693573, 0.07249404489994049, 0.042859554290771484, 0.047397810965776443]\n",
      "extrinsic_reward :  0 intrinsic_reward :  410.0732116699219\n",
      "place 49\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 49, 'x_pos_screen': 49, 'y_pos': 81}\n",
      "global_step :  20 , action :  3 action_prob :  [0.059744659811258316, 0.17647868394851685, 0.04403914138674736, 0.5570842623710632, 0.07245863229036331, 0.04282812401652336, 0.047366611659526825]\n",
      "extrinsic_reward :  1 intrinsic_reward :  409.74365234375\n",
      "place 50\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 50, 'x_pos_screen': 50, 'y_pos': 79}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  21 , action :  3 action_prob :  [0.059839654713869095, 0.17654338479042053, 0.0441221185028553, 0.556575357913971, 0.07255519181489944, 0.04291249439120293, 0.04745174199342728]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.8468322753906\n",
      "place 51\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 51, 'x_pos_screen': 51, 'y_pos': 79}\n",
      "global_step :  22 , action :  3 action_prob :  [0.05985831096768379, 0.1765560656785965, 0.04413874074816704, 0.5564744472503662, 0.07257462292909622, 0.042928896844387054, 0.0474688820540905]\n",
      "extrinsic_reward :  2 intrinsic_reward :  409.69305419921875\n",
      "place 53\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 53, 'x_pos_screen': 53, 'y_pos': 79}\n",
      "global_step :  23 , action :  3 action_prob :  [0.05986681208014488, 0.17656221985816956, 0.04414632171392441, 0.5564278364181519, 0.07258360087871552, 0.04293639212846756, 0.04747675359249115]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.6632995605469\n",
      "place 54\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 54, 'x_pos_screen': 54, 'y_pos': 79}\n",
      "global_step :  24 , action :  3 action_prob :  [0.05987146869301796, 0.17656506597995758, 0.044150177389383316, 0.5564041137695312, 0.07258796691894531, 0.042940545827150345, 0.047480661422014236]\n",
      "extrinsic_reward :  1 intrinsic_reward :  409.3848571777344\n",
      "place 55\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 55, 'x_pos_screen': 55, 'y_pos': 79}\n",
      "global_step :  25 , action :  3 action_prob :  [0.059816379100084305, 0.17652790248394012, 0.044102225452661514, 0.5566982626914978, 0.07253216952085495, 0.042891744524240494, 0.04743138700723648]\n",
      "extrinsic_reward :  2 intrinsic_reward :  410.5209045410156\n",
      "place 57\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 57, 'x_pos_screen': 57, 'y_pos': 79}\n",
      "global_step :  26 , action :  3 action_prob :  [0.05983536317944527, 0.17654086649417877, 0.04411879926919937, 0.5565963387489319, 0.07255152612924576, 0.04290858283638954, 0.04744844138622284]\n",
      "extrinsic_reward :  2 intrinsic_reward :  413.3554382324219\n",
      "place 59\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 59, 'x_pos_screen': 59, 'y_pos': 79}\n",
      "global_step :  27 , action :  0 action_prob :  [0.05986196920275688, 0.1765589863061905, 0.04414183646440506, 0.5564548373222351, 0.07257810235023499, 0.042932406067848206, 0.047471895813941956]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.7680358886719\n",
      "place 61\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 61, 'x_pos_screen': 61, 'y_pos': 79}\n",
      "global_step :  28 , action :  3 action_prob :  [0.05983903259038925, 0.17654331028461456, 0.04412221908569336, 0.5565761923789978, 0.07255560904741287, 0.04291167110204697, 0.04745204374194145]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.5465393066406\n",
      "place 62\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 62, 'x_pos_screen': 62, 'y_pos': 79}\n",
      "global_step :  29 , action :  5 action_prob :  [0.059862274676561356, 0.17655900120735168, 0.044142208993434906, 0.5564531683921814, 0.07257852703332901, 0.04293256625533104, 0.04747231677174568]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.5968017578125\n",
      "place 64\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 64, 'x_pos_screen': 64, 'y_pos': 79}\n",
      "global_step :  30 , action :  3 action_prob :  [0.05982988700270653, 0.17653682827949524, 0.04411410912871361, 0.5566260814666748, 0.0725458562374115, 0.042903754860162735, 0.047443490475416183]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.1564636230469\n",
      "place 66\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 66, 'x_pos_screen': 66, 'y_pos': 85}\n",
      "global_step :  31 , action :  6 action_prob :  [0.059844210743904114, 0.17654652893543243, 0.04412642866373062, 0.5565500855445862, 0.07256009429693222, 0.04291660711169243, 0.04745602607727051]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.43548583984375\n",
      "place 68\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 68, 'x_pos_screen': 68, 'y_pos': 91}\n",
      "global_step :  32 , action :  3 action_prob :  [0.059776682406663895, 0.17650017142295837, 0.04406719654798508, 0.5569131374359131, 0.07249089330434799, 0.042856864631175995, 0.04739506170153618]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.1639709472656\n",
      "place 70\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 70, 'x_pos_screen': 70, 'y_pos': 96}\n",
      "global_step :  33 , action :  3 action_prob :  [0.05982319265604019, 0.1765320897102356, 0.04410744830965996, 0.5566650629043579, 0.07253754884004593, 0.04289843142032623, 0.0474361777305603]\n",
      "extrinsic_reward :  2 intrinsic_reward :  413.2312316894531\n",
      "place 72\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 72, 'x_pos_screen': 72, 'y_pos': 100}\n",
      "global_step :  34 , action :  6 action_prob :  [0.05989044904708862, 0.17657797038555145, 0.044165775179862976, 0.5563064217567444, 0.07260514050722122, 0.04295850172638893, 0.04749579355120659]\n",
      "extrinsic_reward :  2 intrinsic_reward :  414.09124755859375\n",
      "place 74\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 74, 'x_pos_screen': 74, 'y_pos': 104}\n",
      "global_step :  35 , action :  3 action_prob :  [0.059908997267484665, 0.17659181356430054, 0.04418184235692024, 0.556206226348877, 0.07262352108955383, 0.042975734919309616, 0.04751187935471535]\n",
      "extrinsic_reward :  2 intrinsic_reward :  413.56231689453125\n",
      "place 76\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 76, 'x_pos_screen': 76, 'y_pos': 106}\n",
      "global_step :  36 , action :  3 action_prob :  [0.059960488229990005, 0.17662619054317474, 0.04422672465443611, 0.5559318661689758, 0.07267540693283081, 0.043021637946367264, 0.04755770042538643]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.24530029296875\n",
      "place 78\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 78, 'x_pos_screen': 78, 'y_pos': 108}\n",
      "global_step :  37 , action :  3 action_prob :  [0.0599258728325367, 0.17660211026668549, 0.044196829199790955, 0.5561166405677795, 0.07264089584350586, 0.04299040511250496, 0.04752729460597038]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.25689697265625\n",
      "place 80\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 80, 'x_pos_screen': 80, 'y_pos': 109}\n",
      "global_step :  38 , action :  0 action_prob :  [0.06001296266913414, 0.1766619086265564, 0.044272396713495255, 0.5556520223617554, 0.07272771745920181, 0.04306920990347862, 0.047603849321603775]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.5361328125\n",
      "place 82\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 82, 'x_pos_screen': 81, 'y_pos': 109}\n",
      "global_step :  39 , action :  3 action_prob :  [0.0600275844335556, 0.17667075991630554, 0.04428553953766823, 0.5555738210678101, 0.07274313271045685, 0.04308168217539787, 0.04761754348874092]\n",
      "extrinsic_reward :  2 intrinsic_reward :  400.3307800292969\n",
      "place 85\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 85, 'x_pos_screen': 82, 'y_pos': 108}\n",
      "global_step :  40 , action :  3 action_prob :  [0.05875076353549957, 0.17578795552253723, 0.043180353939533234, 0.5623900890350342, 0.07145962119102478, 0.04194077104330063, 0.04649057239294052]\n",
      "extrinsic_reward :  2 intrinsic_reward :  419.2345275878906\n",
      "place 87\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 87, 'x_pos_screen': 83, 'y_pos': 107}\n",
      "global_step :  41 , action :  0 action_prob :  [0.06003599613904953, 0.17666788399219513, 0.04430503770709038, 0.5554937720298767, 0.07277623564004898, 0.043073028326034546, 0.0476481132209301]\n",
      "extrinsic_reward :  3 intrinsic_reward :  408.8414001464844\n",
      "place 90\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 90, 'x_pos_screen': 84, 'y_pos': 104}\n",
      "global_step :  42 , action :  3 action_prob :  [0.05901997908949852, 0.1759667694568634, 0.043423064053058624, 0.5609238743782043, 0.07175247371196747, 0.04216593876481056, 0.04674794524908066]\n",
      "extrinsic_reward :  2 intrinsic_reward :  407.3179016113281\n",
      "place 92\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 92, 'x_pos_screen': 85, 'y_pos': 101}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  43 , action :  0 action_prob :  [0.05975869297981262, 0.1764753758907318, 0.04406150057911873, 0.556989848613739, 0.07249054312705994, 0.042827147990465164, 0.04739689826965332]\n",
      "extrinsic_reward :  3 intrinsic_reward :  419.9659423828125\n",
      "place 95\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 95, 'x_pos_screen': 86, 'y_pos': 97}\n",
      "global_step :  44 , action :  3 action_prob :  [0.06068625673651695, 0.17709366977214813, 0.04486773908138275, 0.5520557165145874, 0.07341953366994858, 0.04365895688533783, 0.04821815341711044]\n",
      "extrinsic_reward :  2 intrinsic_reward :  399.4931335449219\n",
      "place 97\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 97, 'x_pos_screen': 87, 'y_pos': 92}\n",
      "global_step :  45 , action :  3 action_prob :  [0.05847126618027687, 0.17557556927204132, 0.042946331202983856, 0.5638759136199951, 0.07119212299585342, 0.0416802354156971, 0.046258579939603806]\n",
      "extrinsic_reward :  3 intrinsic_reward :  407.45013427734375\n",
      "place 100\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 100, 'x_pos_screen': 88, 'y_pos': 87}\n",
      "global_step :  46 , action :  3 action_prob :  [0.05914139002561569, 0.1760445088148117, 0.04353838413953781, 0.5602445602416992, 0.0718965008854866, 0.04225900024175644, 0.04687562584877014]\n",
      "extrinsic_reward :  3 intrinsic_reward :  415.46368408203125\n",
      "place 103\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 103, 'x_pos_screen': 89, 'y_pos': 82}\n",
      "global_step :  47 , action :  1 action_prob :  [0.05846109986305237, 0.17557254433631897, 0.042942557483911514, 0.5639042258262634, 0.07119396328926086, 0.041666142642498016, 0.04625943675637245]\n",
      "extrinsic_reward :  3 intrinsic_reward :  417.5508728027344\n",
      "place 106\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 106, 'x_pos_screen': 90, 'y_pos': 79}\n",
      "global_step :  48 , action :  3 action_prob :  [0.0591171495616436, 0.17603421211242676, 0.043509211391210556, 0.5603993535041809, 0.07185313105583191, 0.04225056245923042, 0.04683644697070122]\n",
      "extrinsic_reward :  2 intrinsic_reward :  424.464599609375\n",
      "place 108\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 108, 'x_pos_screen': 91, 'y_pos': 79}\n",
      "global_step :  49 , action :  3 action_prob :  [0.05885767191648483, 0.17585565149784088, 0.04328462854027748, 0.5617808103561401, 0.0715935155749321, 0.042019035667181015, 0.046608611941337585]\n",
      "extrinsic_reward :  1 intrinsic_reward :  415.277587890625\n",
      "place 109\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 109, 'x_pos_screen': 91, 'y_pos': 79}\n",
      "global_step :  50 , action :  3 action_prob :  [0.05791651830077171, 0.17518071830272675, 0.04246923699975014, 0.5668386816978455, 0.07063426822423935, 0.04118851199746132, 0.045771997421979904]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.7132568359375\n",
      "place 111\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 111, 'x_pos_screen': 92, 'y_pos': 79}\n",
      "global_step :  51 , action :  0 action_prob :  [0.05830182135105133, 0.17545558512210846, 0.042812902480363846, 0.5647307634353638, 0.07105005532503128, 0.04151391610503197, 0.046134911477565765]\n",
      "extrinsic_reward :  2 intrinsic_reward :  435.7065734863281\n",
      "place 113\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 113, 'x_pos_screen': 93, 'y_pos': 79}\n",
      "global_step :  52 , action :  2 action_prob :  [0.05857638642191887, 0.17565326392650604, 0.043046984821558, 0.5632700324058533, 0.07132058590650558, 0.04176213964819908, 0.04637058079242706]\n",
      "extrinsic_reward :  2 intrinsic_reward :  421.529541015625\n",
      "place 115\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 115, 'x_pos_screen': 94, 'y_pos': 79}\n",
      "global_step :  53 , action :  5 action_prob :  [0.057777393609285355, 0.17507188022136688, 0.042355332523584366, 0.5675705075263977, 0.07050663977861404, 0.04105645418167114, 0.04566182196140289]\n",
      "extrinsic_reward :  2 intrinsic_reward :  435.40625\n",
      "place 117\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 117, 'x_pos_screen': 95, 'y_pos': 85}\n",
      "global_step :  54 , action :  6 action_prob :  [0.0587189644575119, 0.17575834691524506, 0.04316757246851921, 0.5625121593475342, 0.07145918905735016, 0.041891638189554214, 0.0464920848608017]\n",
      "extrinsic_reward :  3 intrinsic_reward :  418.3785400390625\n",
      "place 120\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 120, 'x_pos_screen': 96, 'y_pos': 91}\n",
      "global_step :  55 , action :  3 action_prob :  [0.05834614858031273, 0.17548979818820953, 0.04284116253256798, 0.5645298361778259, 0.07107292115688324, 0.04156558960676193, 0.04615459218621254]\n",
      "extrinsic_reward :  2 intrinsic_reward :  420.4093017578125\n",
      "place 122\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 122, 'x_pos_screen': 97, 'y_pos': 97}\n",
      "global_step :  56 , action :  3 action_prob :  [0.05803031101822853, 0.17526096105575562, 0.04257277399301529, 0.5662088394165039, 0.07076193392276764, 0.04128149524331093, 0.045883674174547195]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.5269470214844\n",
      "place 124\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 124, 'x_pos_screen': 98, 'y_pos': 102}\n",
      "global_step :  57 , action :  3 action_prob :  [0.05737372115254402, 0.17477433383464813, 0.04200344905257225, 0.5697590112686157, 0.07008614391088486, 0.04070558026432991, 0.045297689735889435]\n",
      "extrinsic_reward :  2 intrinsic_reward :  427.9507141113281\n",
      "place 126\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 126, 'x_pos_screen': 99, 'y_pos': 106}\n",
      "global_step :  58 , action :  3 action_prob :  [0.05752532184123993, 0.17489008605480194, 0.042143579572439194, 0.5688998103141785, 0.07026269286870956, 0.040827807039022446, 0.045450638979673386]\n",
      "extrinsic_reward :  2 intrinsic_reward :  436.5148620605469\n",
      "place 128\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 128, 'x_pos_screen': 100, 'y_pos': 109}\n",
      "global_step :  59 , action :  3 action_prob :  [0.057778116315603256, 0.17507819831371307, 0.04235757887363434, 0.5675523281097412, 0.0705118477344513, 0.041055548936128616, 0.04566643014550209]\n",
      "extrinsic_reward :  2 intrinsic_reward :  426.05804443359375\n",
      "place 131\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 131, 'x_pos_screen': 101, 'y_pos': 111}\n",
      "global_step :  60 , action :  3 action_prob :  [0.05710223689675331, 0.17457525432109833, 0.04177304729819298, 0.5712018609046936, 0.0698186531662941, 0.0404634103178978, 0.04506553336977959]\n",
      "extrinsic_reward :  2 intrinsic_reward :  423.55511474609375\n",
      "place 133\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 133, 'x_pos_screen': 102, 'y_pos': 112}\n",
      "global_step :  61 , action :  3 action_prob :  [0.05782775953412056, 0.17510826885700226, 0.042396824806928635, 0.5673086643218994, 0.07055274397134781, 0.041103463619947433, 0.04570234194397926]\n",
      "extrinsic_reward :  3 intrinsic_reward :  429.6490783691406\n",
      "place 136\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 136, 'x_pos_screen': 103, 'y_pos': 113}\n",
      "global_step :  62 , action :  3 action_prob :  [0.058374565094709396, 0.17550332844257355, 0.04287116602063179, 0.564365565776825, 0.07111193239688873, 0.04158369451761246, 0.04618985205888748]\n",
      "extrinsic_reward :  3 intrinsic_reward :  422.50018310546875\n",
      "place 139\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 139, 'x_pos_screen': 104, 'y_pos': 113}\n",
      "global_step :  63 , action :  3 action_prob :  [0.05675893649458885, 0.1743173599243164, 0.04147232696413994, 0.5730738639831543, 0.06945627182722092, 0.040168460458517075, 0.044752754271030426]\n",
      "extrinsic_reward :  2 intrinsic_reward :  432.980712890625\n",
      "place 141\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 141, 'x_pos_screen': 105, 'y_pos': 112}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  64 , action :  3 action_prob :  [0.05714671313762665, 0.174601748585701, 0.041814640164375305, 0.5709572434425354, 0.0698714479804039, 0.040495578199625015, 0.045112691819667816]\n",
      "extrinsic_reward :  3 intrinsic_reward :  428.06732177734375\n",
      "place 144\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 144, 'x_pos_screen': 106, 'y_pos': 110}\n",
      "global_step :  65 , action :  3 action_prob :  [0.05670202150940895, 0.17425698041915894, 0.04143271595239639, 0.5733663439750671, 0.06941680610179901, 0.04010433703660965, 0.04472067952156067]\n",
      "extrinsic_reward :  3 intrinsic_reward :  421.8553466796875\n",
      "place 147\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 147, 'x_pos_screen': 107, 'y_pos': 108}\n",
      "global_step :  66 , action :  3 action_prob :  [0.057204511016607285, 0.17464035749435425, 0.04186017066240311, 0.5706691145896912, 0.06992007046937943, 0.040550775825977325, 0.045154962688684464]\n",
      "extrinsic_reward :  3 intrinsic_reward :  425.12908935546875\n",
      "place 150\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 150, 'x_pos_screen': 108, 'y_pos': 104}\n",
      "global_step :  67 , action :  2 action_prob :  [0.056803181767463684, 0.1743372082710266, 0.04152032360434532, 0.5728128552436829, 0.06952285766601562, 0.04019109159708023, 0.0448123961687088]\n",
      "extrinsic_reward :  3 intrinsic_reward :  427.4707946777344\n",
      "place 153\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 153, 'x_pos_screen': 109, 'y_pos': 100}\n",
      "global_step :  68 , action :  2 action_prob :  [0.056264884769916534, 0.17391569912433624, 0.04106248542666435, 0.5757164359092712, 0.06898055225610733, 0.0397135354578495, 0.044346436858177185]\n",
      "extrinsic_reward :  3 intrinsic_reward :  437.07421875\n",
      "place 156\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 156, 'x_pos_screen': 110, 'y_pos': 95}\n",
      "global_step :  69 , action :  3 action_prob :  [0.056078918278217316, 0.17377342283725739, 0.04089682176709175, 0.5767432451248169, 0.06877783685922623, 0.03955812752246857, 0.044171638786792755]\n",
      "extrinsic_reward :  3 intrinsic_reward :  433.6528015136719\n",
      "place 159\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 159, 'x_pos_screen': 111, 'y_pos': 90}\n",
      "global_step :  70 , action :  1 action_prob :  [0.05669368430972099, 0.17424163222312927, 0.04142381623387337, 0.5734304785728455, 0.06940395385026932, 0.04009530320763588, 0.04471120238304138]\n",
      "extrinsic_reward :  3 intrinsic_reward :  436.17303466796875\n",
      "place 162\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 162, 'x_pos_screen': 112, 'y_pos': 85}\n",
      "global_step :  71 , action :  4 action_prob :  [0.05622929707169533, 0.17388758063316345, 0.04102782532572746, 0.5759247541427612, 0.06893672794103622, 0.03968429937958717, 0.04430951550602913]\n",
      "extrinsic_reward :  3 intrinsic_reward :  435.93829345703125\n",
      "place 165\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 165, 'x_pos_screen': 112, 'y_pos': 80}\n",
      "global_step :  72 , action :  6 action_prob :  [0.05573587119579315, 0.17348319292068481, 0.04061104729771614, 0.578599214553833, 0.06844035536050797, 0.03924494609236717, 0.0438854917883873]\n",
      "extrinsic_reward :  3 intrinsic_reward :  445.3832092285156\n",
      "place 168\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 168, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  73 , action :  4 action_prob :  [0.056036777794361115, 0.17372673749923706, 0.0408640019595623, 0.5769786834716797, 0.06873872131109238, 0.039514537900686264, 0.04414045438170433]\n",
      "extrinsic_reward :  3 intrinsic_reward :  433.6063232421875\n",
      "place 171\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 171, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  74 , action :  6 action_prob :  [0.05570153892040253, 0.17346718907356262, 0.04057816416025162, 0.5787844657897949, 0.06839890033006668, 0.03922070935368538, 0.04384908080101013]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.9523010253906\n",
      "place 174\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 174, 'x_pos_screen': 112, 'y_pos': 85}\n",
      "global_step :  75 , action :  4 action_prob :  [0.0554497167468071, 0.17326924204826355, 0.04035947844386101, 0.580161452293396, 0.06813260912895203, 0.039007123559713364, 0.04362040385603905]\n",
      "extrinsic_reward :  3 intrinsic_reward :  443.63006591796875\n",
      "place 177\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 177, 'x_pos_screen': 112, 'y_pos': 91}\n",
      "global_step :  76 , action :  1 action_prob :  [0.05559530481696129, 0.17338889837265015, 0.0404789000749588, 0.579386293888092, 0.06827154010534286, 0.0391402430832386, 0.04373878240585327]\n",
      "extrinsic_reward :  3 intrinsic_reward :  450.7735290527344\n",
      "place 180\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 180, 'x_pos_screen': 112, 'y_pos': 96}\n",
      "global_step :  77 , action :  2 action_prob :  [0.05527014285326004, 0.17314888536930084, 0.04019850492477417, 0.5811354517936707, 0.06793424487113953, 0.03886404633522034, 0.04344874620437622]\n",
      "extrinsic_reward :  3 intrinsic_reward :  449.57867431640625\n",
      "place 183\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 183, 'x_pos_screen': 112, 'y_pos': 100}\n",
      "global_step :  78 , action :  1 action_prob :  [0.054967001080513, 0.1729011833667755, 0.03994205594062805, 0.5827785134315491, 0.06762579828500748, 0.03860000520944595, 0.04318536818027496]\n",
      "extrinsic_reward :  3 intrinsic_reward :  446.944091796875\n",
      "place 186\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 186, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  79 , action :  3 action_prob :  [0.055384065955877304, 0.1732463836669922, 0.040295884013175964, 0.580511212348938, 0.06804952025413513, 0.038966502994298935, 0.043546415865421295]\n",
      "extrinsic_reward :  2 intrinsic_reward :  444.408203125\n",
      "place 189\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 189, 'x_pos_screen': 112, 'y_pos': 106}\n",
      "global_step :  80 , action :  1 action_prob :  [0.05466919019818306, 0.17265547811985016, 0.03969211131334305, 0.584388256072998, 0.0673261284828186, 0.038337744772434235, 0.04293104261159897]\n",
      "extrinsic_reward :  3 intrinsic_reward :  450.91900634765625\n",
      "place 192\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 192, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  81 , action :  3 action_prob :  [0.05484688654541969, 0.17282365262508392, 0.0398334339261055, 0.5834310054779053, 0.06748989224433899, 0.038507748395204544, 0.043067336082458496]\n",
      "extrinsic_reward :  3 intrinsic_reward :  452.58355712890625\n",
      "place 195\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 195, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  82 , action :  3 action_prob :  [0.054619502276182175, 0.17263425886631012, 0.03964358940720558, 0.5846585035324097, 0.06726299971342087, 0.03830565884709358, 0.042875487357378006]\n",
      "extrinsic_reward :  3 intrinsic_reward :  453.7254638671875\n",
      "place 198\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 198, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  83 , action :  5 action_prob :  [0.054744087159633636, 0.17274311184883118, 0.03974432125687599, 0.5839928984642029, 0.06737994402647018, 0.038422390818595886, 0.04297318309545517]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.82403564453125\n",
      "place 201\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 201, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  84 , action :  1 action_prob :  [0.054869119077920914, 0.17285409569740295, 0.03984406962990761, 0.5833276510238647, 0.06749475002288818, 0.03854093328118324, 0.04306938871741295]\n",
      "extrinsic_reward :  3 intrinsic_reward :  456.3824157714844\n",
      "place 204\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 204, 'x_pos_screen': 112, 'y_pos': 107}\n",
      "global_step :  85 , action :  1 action_prob :  [0.054893992841243744, 0.1728745996952057, 0.0398697666823864, 0.5831748247146606, 0.06752998381853104, 0.038556549698114395, 0.04310021549463272]\n",
      "extrinsic_reward :  3 intrinsic_reward :  451.18939208984375\n",
      "place 207\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 207, 'x_pos_screen': 112, 'y_pos': 104}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  86 , action :  3 action_prob :  [0.05457090958952904, 0.17261597514152527, 0.039589665830135345, 0.584944486618042, 0.06718721985816956, 0.038283947855234146, 0.042807694524526596]\n",
      "extrinsic_reward :  3 intrinsic_reward :  450.08953857421875\n",
      "place 210\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 210, 'x_pos_screen': 112, 'y_pos': 101}\n",
      "global_step :  87 , action :  3 action_prob :  [0.05509524047374725, 0.17303602397441864, 0.040041107684373856, 0.582086443901062, 0.06773441284894943, 0.03873194381594658, 0.043274879455566406]\n",
      "extrinsic_reward :  3 intrinsic_reward :  446.8307189941406\n",
      "place 213\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 213, 'x_pos_screen': 112, 'y_pos': 97}\n",
      "global_step :  88 , action :  3 action_prob :  [0.05430394411087036, 0.17238500714302063, 0.03936558589339256, 0.5863998532295227, 0.06691815704107285, 0.038046084344387054, 0.04258127883076668]\n",
      "extrinsic_reward :  3 intrinsic_reward :  455.04693603515625\n",
      "place 216\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 216, 'x_pos_screen': 112, 'y_pos': 92}\n",
      "global_step :  89 , action :  6 action_prob :  [0.05491047725081444, 0.1728888601064682, 0.03987574577331543, 0.5831152200698853, 0.0675303116440773, 0.038579098880290985, 0.04310028254985809]\n",
      "extrinsic_reward :  3 intrinsic_reward :  445.02392578125\n",
      "place 219\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 219, 'x_pos_screen': 112, 'y_pos': 87}\n",
      "global_step :  90 , action :  3 action_prob :  [0.05458132550120354, 0.17262011766433716, 0.0395984947681427, 0.5848933458328247, 0.06719839572906494, 0.03828916326165199, 0.0428190715610981]\n",
      "extrinsic_reward :  3 intrinsic_reward :  448.3961181640625\n",
      "place 222\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 222, 'x_pos_screen': 112, 'y_pos': 82}\n",
      "global_step :  91 , action :  3 action_prob :  [0.05502473562955856, 0.17295868694782257, 0.03997703641653061, 0.5825090408325195, 0.06765361875295639, 0.038668859750032425, 0.04320799186825752]\n",
      "extrinsic_reward :  3 intrinsic_reward :  444.6025085449219\n",
      "place 225\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 225, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  92 , action :  4 action_prob :  [0.055344320833683014, 0.1732175648212433, 0.04024488851428032, 0.5807885527610779, 0.06797321885824203, 0.038951627910137177, 0.04347986727952957]\n",
      "extrinsic_reward :  3 intrinsic_reward :  445.64874267578125\n",
      "place 228\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 228, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  93 , action :  3 action_prob :  [0.05519434064626694, 0.17309963703155518, 0.0401252806186676, 0.5815680027008057, 0.06783656030893326, 0.03881246596574783, 0.04336372762918472]\n",
      "extrinsic_reward :  3 intrinsic_reward :  458.66668701171875\n",
      "place 231\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 231, 'x_pos_screen': 112, 'y_pos': 85}\n",
      "global_step :  94 , action :  3 action_prob :  [0.055348411202430725, 0.17319606244564056, 0.040258944034576416, 0.5807565450668335, 0.06799855083227158, 0.03893681615591049, 0.043504659086465836]\n",
      "extrinsic_reward :  3 intrinsic_reward :  446.8981018066406\n",
      "place 234\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 234, 'x_pos_screen': 112, 'y_pos': 91}\n",
      "global_step :  95 , action :  3 action_prob :  [0.055511727929115295, 0.17334118485450745, 0.04039795696735382, 0.579853892326355, 0.06816640496253967, 0.03908141702413559, 0.043647367507219315]\n",
      "extrinsic_reward :  3 intrinsic_reward :  451.77984619140625\n",
      "place 237\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 237, 'x_pos_screen': 112, 'y_pos': 96}\n",
      "global_step :  96 , action :  3 action_prob :  [0.054904814809560776, 0.1728394329547882, 0.039881572127342224, 0.5831595063209534, 0.06754592806100845, 0.03854978084564209, 0.04311903193593025]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.59100341796875\n",
      "place 240\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 240, 'x_pos_screen': 112, 'y_pos': 100}\n",
      "global_step :  97 , action :  3 action_prob :  [0.05565240979194641, 0.17343316972255707, 0.04051835089921951, 0.579113245010376, 0.06831128895282745, 0.03919864818453789, 0.04377280920743942]\n",
      "extrinsic_reward :  3 intrinsic_reward :  443.49603271484375\n",
      "place 243\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 243, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  98 , action :  3 action_prob :  [0.05532693490386009, 0.17317941784858704, 0.040244050323963165, 0.58085697889328, 0.0679858922958374, 0.038912076503038406, 0.04349464550614357]\n",
      "extrinsic_reward :  3 intrinsic_reward :  450.0396728515625\n",
      "place 246\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 246, 'x_pos_screen': 112, 'y_pos': 106}\n",
      "global_step :  99 , action :  1 action_prob :  [0.0556563101708889, 0.17343129217624664, 0.040523551404476166, 0.5790895223617554, 0.06832024455070496, 0.03919802978634834, 0.04378102719783783]\n",
      "extrinsic_reward :  2 intrinsic_reward :  442.980224609375\n",
      "place 249\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 249, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  100 , action :  3 action_prob :  [0.05600965768098831, 0.1737077683210373, 0.04082166776061058, 0.5771942138671875, 0.06867270916700363, 0.0395115464925766, 0.04408241808414459]\n",
      "extrinsic_reward :  3 intrinsic_reward :  445.3555908203125\n",
      "place 252\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 252, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  101 , action :  3 action_prob :  [0.05570419132709503, 0.17347650229930878, 0.04056571051478386, 0.5788153409957886, 0.06837298721075058, 0.03923999145627022, 0.04382526874542236]\n",
      "extrinsic_reward :  3 intrinsic_reward :  459.2484130859375\n",
      "place 255\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 255, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  102 , action :  0 action_prob :  [0.0557648241519928, 0.17351041734218597, 0.0406157523393631, 0.5785101056098938, 0.06843164563179016, 0.039290644228458405, 0.04387662932276726]\n",
      "extrinsic_reward :  3 intrinsic_reward :  453.25238037109375\n",
      "place 258\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 258, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  103 , action :  3 action_prob :  [0.05588464066386223, 0.17361649870872498, 0.04071884974837303, 0.57784503698349, 0.06855572760105133, 0.039396993815898895, 0.04398231580853462]\n",
      "extrinsic_reward :  3 intrinsic_reward :  451.6719665527344\n",
      "place 261\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 261, 'x_pos_screen': 112, 'y_pos': 107}\n",
      "global_step :  104 , action :  3 action_prob :  [0.055191438645124435, 0.17306049168109894, 0.040125783532857895, 0.5816117525100708, 0.06784352660179138, 0.03879428282380104, 0.04337279871106148]\n",
      "extrinsic_reward :  3 intrinsic_reward :  446.93829345703125\n",
      "place 264\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 264, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  105 , action :  1 action_prob :  [0.05560806393623352, 0.17340201139450073, 0.04047555476427078, 0.5793619155883789, 0.06826087832450867, 0.039163313806056976, 0.043728336691856384]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.7178955078125\n",
      "place 267\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 267, 'x_pos_screen': 112, 'y_pos': 101}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  106 , action :  3 action_prob :  [0.055427830666303635, 0.1732507199048996, 0.0403253436088562, 0.5803360342979431, 0.06808213889598846, 0.039002031087875366, 0.0435759536921978]\n",
      "extrinsic_reward :  3 intrinsic_reward :  453.859130859375\n",
      "place 270\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 270, 'x_pos_screen': 112, 'y_pos': 97}\n",
      "global_step :  107 , action :  3 action_prob :  [0.055407360196113586, 0.17323730885982513, 0.04030415788292885, 0.5804594159126282, 0.0680517703294754, 0.03899035230278969, 0.04354958236217499]\n",
      "extrinsic_reward :  3 intrinsic_reward :  444.97003173828125\n",
      "place 273\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 273, 'x_pos_screen': 112, 'y_pos': 92}\n",
      "global_step :  108 , action :  3 action_prob :  [0.055643919855356216, 0.17342299222946167, 0.04050181806087494, 0.5791960954666138, 0.06828563660383224, 0.03920040279626846, 0.043749041855335236]\n",
      "extrinsic_reward :  2 intrinsic_reward :  439.3520202636719\n",
      "place 275\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 275, 'x_pos_screen': 111, 'y_pos': 87}\n",
      "global_step :  109 , action :  3 action_prob :  [0.055240925401449203, 0.17310045659542084, 0.04016299545764923, 0.5813634395599365, 0.06788290292024612, 0.038843609392642975, 0.043405599892139435]\n",
      "extrinsic_reward :  -1 intrinsic_reward :  442.9366455078125\n",
      "place 274\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 274, 'x_pos_screen': 110, 'y_pos': 82}\n",
      "global_step :  110 , action :  3 action_prob :  [0.05504259094595909, 0.17294442653656006, 0.03998943045735359, 0.582456111907959, 0.0676664412021637, 0.038681063801050186, 0.04321986436843872]\n",
      "extrinsic_reward :  0 intrinsic_reward :  441.040771484375\n",
      "place 274\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 274, 'x_pos_screen': 110, 'y_pos': 77}\n",
      "global_step :  111 , action :  3 action_prob :  [0.0550273098051548, 0.17292895913124084, 0.03997718542814255, 0.5825403332710266, 0.06765197962522507, 0.03866635635495186, 0.04320789873600006]\n",
      "extrinsic_reward :  0 intrinsic_reward :  442.6933898925781\n",
      "place 274\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 274, 'x_pos_screen': 110, 'y_pos': 72}\n",
      "global_step :  112 , action :  3 action_prob :  [0.05509188398718834, 0.17298175394535065, 0.040034253150224686, 0.5821818709373474, 0.06772179901599884, 0.03872125968337059, 0.04326724633574486]\n",
      "extrinsic_reward :  0 intrinsic_reward :  443.4818420410156\n",
      "place 274\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 274, 'x_pos_screen': 110, 'y_pos': 67}\n",
      "global_step :  113 , action :  3 action_prob :  [0.05504002422094345, 0.1729428619146347, 0.039988450706005096, 0.5824649333953857, 0.06766614317893982, 0.03867805004119873, 0.04321944713592529]\n",
      "extrinsic_reward :  0 intrinsic_reward :  443.8682861328125\n",
      "place 274\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 274, 'x_pos_screen': 110, 'y_pos': 62}\n",
      "global_step :  114 , action :  0 action_prob :  [0.05505591258406639, 0.17296013236045837, 0.04000112786889076, 0.5823764204978943, 0.06768090277910233, 0.038694046437740326, 0.04323141276836395]\n",
      "extrinsic_reward :  1 intrinsic_reward :  446.868896484375\n",
      "place 275\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 275, 'x_pos_screen': 111, 'y_pos': 57}\n",
      "global_step :  115 , action :  0 action_prob :  [0.05503223091363907, 0.1729438304901123, 0.03997964784502983, 0.5825063586235046, 0.06765401363372803, 0.03867582604289055, 0.04320801794528961]\n",
      "extrinsic_reward :  0 intrinsic_reward :  445.60174560546875\n",
      "place 275\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 275, 'x_pos_screen': 111, 'y_pos': 52}\n",
      "global_step :  116 , action :  3 action_prob :  [0.05502328649163246, 0.17293573915958405, 0.03997177258133888, 0.5825564861297607, 0.06764477491378784, 0.03866758197546005, 0.0432002879679203]\n",
      "extrinsic_reward :  1 intrinsic_reward :  445.311767578125\n",
      "place 276\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 276, 'x_pos_screen': 112, 'y_pos': 47}\n",
      "global_step :  117 , action :  2 action_prob :  [0.05502999573945999, 0.17294088006019592, 0.039977408945560455, 0.5825208425521851, 0.0676514133810997, 0.03867359831929207, 0.0432058647274971]\n",
      "extrinsic_reward :  0 intrinsic_reward :  445.75299072265625\n",
      "place 276\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 276, 'x_pos_screen': 112, 'y_pos': 42}\n",
      "global_step :  118 , action :  3 action_prob :  [0.05501221865415573, 0.17292626202106476, 0.03996211290359497, 0.5826180577278137, 0.06763318181037903, 0.03865776211023331, 0.04319051653146744]\n",
      "extrinsic_reward :  1 intrinsic_reward :  445.4691162109375\n",
      "place 277\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 277, 'x_pos_screen': 113, 'y_pos': 37}\n",
      "global_step :  119 , action :  3 action_prob :  [0.05499403923749924, 0.1729109287261963, 0.03994663059711456, 0.5827170610427856, 0.06761486083269119, 0.038641344755887985, 0.043175093829631805]\n",
      "extrinsic_reward :  -1 intrinsic_reward :  445.370361328125\n",
      "place 277\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 277, 'x_pos_screen': 113, 'y_pos': 32}\n",
      "global_step :  120 , action :  3 action_prob :  [0.054993122816085815, 0.17291007936000824, 0.03994588926434517, 0.5827219486236572, 0.06761401146650314, 0.03864045441150665, 0.043174415826797485]\n",
      "extrinsic_reward :  1 intrinsic_reward :  445.1861572265625\n",
      "place 278\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 278, 'x_pos_screen': 114, 'y_pos': 27}\n",
      "global_step :  121 , action :  3 action_prob :  [0.05498176068067551, 0.17290125787258148, 0.03993618115782738, 0.5827834010124207, 0.06760236620903015, 0.03863061964511871, 0.043164487928152084]\n",
      "extrinsic_reward :  1 intrinsic_reward :  445.0549011230469\n",
      "place 279\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 279, 'x_pos_screen': 115, 'y_pos': 22}\n",
      "global_step :  122 , action :  4 action_prob :  [0.05497894436120987, 0.1728990226984024, 0.03993373364210129, 0.5827988386154175, 0.06759940087795258, 0.038628216832876205, 0.04316195473074913]\n",
      "extrinsic_reward :  1 intrinsic_reward :  445.0265808105469\n",
      "place 280\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 280, 'x_pos_screen': 116, 'y_pos': 17}\n",
      "global_step :  123 , action :  3 action_prob :  [0.05498379468917847, 0.17290282249450684, 0.03993773087859154, 0.582772970199585, 0.06760403513908386, 0.038632627576589584, 0.04316588491201401]\n",
      "extrinsic_reward :  1 intrinsic_reward :  445.15008544921875\n",
      "place 281\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 281, 'x_pos_screen': 117, 'y_pos': 12}\n",
      "global_step :  124 , action :  5 action_prob :  [0.05498276650905609, 0.17290201783180237, 0.03993692249059677, 0.5827785134315491, 0.06760311871767044, 0.038631659001111984, 0.04316510260105133]\n",
      "extrinsic_reward :  1 intrinsic_reward :  445.1202697753906\n",
      "place 282\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 282, 'x_pos_screen': 118, 'y_pos': 7}\n",
      "global_step :  125 , action :  3 action_prob :  [0.05498258396983147, 0.17290183901786804, 0.039936747401952744, 0.5827794671058655, 0.06760291010141373, 0.03863150626420975, 0.043164923787117004]\n",
      "extrinsic_reward :  1 intrinsic_reward :  445.1202697753906\n",
      "place 283\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 283, 'x_pos_screen': 118, 'y_pos': 2}\n",
      "epoch :  0 , global_step :  126\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  1 , action :  6 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.56402587890625\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  2 , action :  3 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  -1 intrinsic_reward :  411.56402587890625\n",
      "place 39\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 39, 'x_pos_screen': 39, 'y_pos': 79}\n",
      "global_step :  3 , action :  3 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.9801025390625\n",
      "place 39\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 39, 'x_pos_screen': 39, 'y_pos': 79}\n",
      "global_step :  4 , action :  3 action_prob :  [0.05977978557348251, 0.17650000751018524, 0.044070955365896225, 0.5568943619728088, 0.07249711453914642, 0.04285694286227226, 0.04740080609917641]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.994384765625\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  5 , action :  4 action_prob :  [0.05977524444460869, 0.17649653553962708, 0.044067103415727615, 0.556918740272522, 0.07249278575181961, 0.04285263642668724, 0.04739701747894287]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.946533203125\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  6 , action :  3 action_prob :  [0.05979273095726967, 0.1765090972185135, 0.04408237338066101, 0.5568243265151978, 0.0725105032324791, 0.04286839812994003, 0.04741258919239044]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.946533203125\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 84}\n",
      "global_step :  7 , action :  4 action_prob :  [0.05979273095726967, 0.1765090972185135, 0.04408237338066101, 0.5568243265151978, 0.0725105032324791, 0.04286839812994003, 0.04741258919239044]\n",
      "extrinsic_reward :  1 intrinsic_reward :  413.3307800292969\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 89}\n",
      "global_step :  8 , action :  1 action_prob :  [0.05978989228606224, 0.1765068620443344, 0.04407934099435806, 0.5568422079086304, 0.07250624150037766, 0.04286649823188782, 0.04740886762738228]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.4372253417969\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 93}\n",
      "global_step :  9 , action :  3 action_prob :  [0.05978158861398697, 0.1765023171901703, 0.04407185688614845, 0.5568862557411194, 0.07249730080366135, 0.04285978898406029, 0.04740084335207939]\n",
      "extrinsic_reward :  0 intrinsic_reward :  410.5800476074219\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 96}\n",
      "global_step :  10 , action :  1 action_prob :  [0.05967511236667633, 0.17643098533153534, 0.043978311121463776, 0.5574571490287781, 0.07238789647817612, 0.042766306549310684, 0.047304295003414154]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.41595458984375\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 99}\n",
      "global_step :  11 , action :  1 action_prob :  [0.05970977991819382, 0.1764560043811798, 0.04400821402668953, 0.5572711229324341, 0.07242247462272644, 0.04279779642820358, 0.04733460023999214]\n",
      "extrinsic_reward :  1 intrinsic_reward :  413.92095947265625\n",
      "place 43\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 43, 'x_pos_screen': 43, 'y_pos': 101}\n",
      "global_step :  12 , action :  1 action_prob :  [0.059668730944395065, 0.17642799019813538, 0.04397206008434296, 0.5574920773506165, 0.07238011062145233, 0.042761653661727905, 0.04729732498526573]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.4979248046875\n",
      "place 43\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 43, 'x_pos_screen': 43, 'y_pos': 102}\n",
      "global_step :  13 , action :  3 action_prob :  [0.05972151830792427, 0.17646381258964539, 0.04401812329888344, 0.5572099089622498, 0.07243361324071884, 0.04280855506658554, 0.04734447970986366]\n",
      "extrinsic_reward :  1 intrinsic_reward :  413.6655578613281\n",
      "place 44\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 44, 'x_pos_screen': 44, 'y_pos': 103}\n",
      "global_step :  14 , action :  3 action_prob :  [0.05974475294351578, 0.17647983133792877, 0.04403827339410782, 0.5570858120918274, 0.07245691120624542, 0.04282945767045021, 0.0473649837076664]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.5640869140625\n",
      "place 45\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 45, 'x_pos_screen': 45, 'y_pos': 103}\n",
      "global_step :  15 , action :  4 action_prob :  [0.05971227586269379, 0.17645803093910217, 0.04400961473584175, 0.5572603940963745, 0.07242336124181747, 0.04280100017786026, 0.04733536019921303]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.1264343261719\n",
      "place 46\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 46, 'x_pos_screen': 46, 'y_pos': 102}\n",
      "global_step :  16 , action :  1 action_prob :  [0.059769850224256516, 0.17649728059768677, 0.044060103595256805, 0.5569513440132141, 0.07248226553201675, 0.04285189509391785, 0.047387294471263885]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.8313903808594\n",
      "place 47\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 47, 'x_pos_screen': 47, 'y_pos': 101}\n",
      "global_step :  17 , action :  0 action_prob :  [0.05976053699851036, 0.17649036645889282, 0.04405229538679123, 0.5570008158683777, 0.07247339934110641, 0.04284314811229706, 0.04737955331802368]\n",
      "extrinsic_reward :  1 intrinsic_reward :  409.9421081542969\n",
      "place 48\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 48, 'x_pos_screen': 48, 'y_pos': 99}\n",
      "global_step :  18 , action :  1 action_prob :  [0.05979563295841217, 0.17651434242725372, 0.044083092361688614, 0.5568122863769531, 0.07250934839248657, 0.04287412017583847, 0.047411270439624786]\n",
      "extrinsic_reward :  1 intrinsic_reward :  408.4043884277344\n",
      "place 49\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 49, 'x_pos_screen': 49, 'y_pos': 96}\n",
      "global_step :  19 , action :  4 action_prob :  [0.059745997190475464, 0.17648042738437653, 0.04403994604945183, 0.5570773482322693, 0.07245934754610062, 0.042829789221286774, 0.04736723005771637]\n",
      "extrinsic_reward :  0 intrinsic_reward :  409.71978759765625\n",
      "place 50\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 50, 'x_pos_screen': 50, 'y_pos': 93}\n",
      "global_step :  20 , action :  6 action_prob :  [0.059788499027490616, 0.17650963366031647, 0.04407721385359764, 0.5568488836288452, 0.07250278443098068, 0.042867500334978104, 0.047405488789081573]\n",
      "extrinsic_reward :  1 intrinsic_reward :  409.5574645996094\n",
      "place 51\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 51, 'x_pos_screen': 51, 'y_pos': 89}\n",
      "global_step :  21 , action :  0 action_prob :  [0.05971686542034149, 0.1764606386423111, 0.04401404410600662, 0.557235062122345, 0.07242865115404129, 0.04280462488532066, 0.04734012857079506]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.2605895996094\n",
      "place 52\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 52, 'x_pos_screen': 52, 'y_pos': 85}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  22 , action :  2 action_prob :  [0.059823594987392426, 0.17653338611125946, 0.04410785809159279, 0.5566612482070923, 0.0725383311510086, 0.042898811399936676, 0.047436781227588654]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.3907470703125\n",
      "place 53\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 53, 'x_pos_screen': 53, 'y_pos': 79}\n",
      "global_step :  23 , action :  3 action_prob :  [0.059806082397699356, 0.17652112245559692, 0.044092871248722076, 0.5567542910575867, 0.07252106815576553, 0.042882952839136124, 0.047421593219041824]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.38330078125\n",
      "place 54\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 54, 'x_pos_screen': 54, 'y_pos': 84}\n",
      "global_step :  24 , action :  1 action_prob :  [0.05984826758503914, 0.17654940485954285, 0.04413039982318878, 0.5565264821052551, 0.07256525009870529, 0.04291965812444687, 0.04746057465672493]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.1415710449219\n",
      "place 55\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 55, 'x_pos_screen': 55, 'y_pos': 89}\n",
      "global_step :  25 , action :  3 action_prob :  [0.05984755605459213, 0.17654924094676971, 0.04412870481610298, 0.556533932685852, 0.07256244868040085, 0.04292002320289612, 0.04745814576745033]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.1191711425781\n",
      "place 57\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 57, 'x_pos_screen': 57, 'y_pos': 93}\n",
      "global_step :  26 , action :  1 action_prob :  [0.05986544117331505, 0.1765616536140442, 0.04414430260658264, 0.5564379096031189, 0.07258055359125137, 0.042936068028211594, 0.04747406020760536]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.03875732421875\n",
      "place 58\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 58, 'x_pos_screen': 58, 'y_pos': 96}\n",
      "global_step :  27 , action :  0 action_prob :  [0.05979998782277107, 0.17651709914207458, 0.04408715292811394, 0.5567883253097534, 0.07251402735710144, 0.042877957224845886, 0.047415412962436676]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.7388000488281\n",
      "place 59\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 59, 'x_pos_screen': 59, 'y_pos': 99}\n",
      "global_step :  28 , action :  2 action_prob :  [0.05984748154878616, 0.17654994130134583, 0.0441284216940403, 0.5565341711044312, 0.07256193459033966, 0.04292048513889313, 0.04745756462216377]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.9879455566406\n",
      "place 61\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 61, 'x_pos_screen': 61, 'y_pos': 101}\n",
      "global_step :  29 , action :  3 action_prob :  [0.05984931066632271, 0.17655052244663239, 0.04413025453686714, 0.556524395942688, 0.0725642591714859, 0.042921602725982666, 0.04745972156524658]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.51947021484375\n",
      "place 62\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 62, 'x_pos_screen': 62, 'y_pos': 102}\n",
      "global_step :  30 , action :  3 action_prob :  [0.0598016157746315, 0.176518976688385, 0.04408780857920647, 0.5567814707756042, 0.07251422107219696, 0.04288039356470108, 0.04741549864411354]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.35455322265625\n",
      "place 64\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 64, 'x_pos_screen': 64, 'y_pos': 103}\n",
      "global_step :  31 , action :  3 action_prob :  [0.05978699401021004, 0.17650875449180603, 0.04407510906457901, 0.5568598508834839, 0.07249949872493744, 0.0428672730922699, 0.04740253835916519]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.89508056640625\n",
      "place 65\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 65, 'x_pos_screen': 65, 'y_pos': 103}\n",
      "global_step :  32 , action :  6 action_prob :  [0.05975184962153435, 0.17648358643054962, 0.04404496029019356, 0.5570475459098816, 0.07246482372283936, 0.04283507540822029, 0.04737216234207153]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.0343017578125\n",
      "place 67\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 67, 'x_pos_screen': 67, 'y_pos': 102}\n",
      "global_step :  33 , action :  3 action_prob :  [0.05984395742416382, 0.17654719948768616, 0.04412534460425377, 0.5565535426139832, 0.07255835831165314, 0.0429171584546566, 0.04745452478528023]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.55926513671875\n",
      "place 68\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 68, 'x_pos_screen': 68, 'y_pos': 101}\n",
      "global_step :  34 , action :  3 action_prob :  [0.05986566096544266, 0.1765623390674591, 0.044143788516521454, 0.5564387440681458, 0.07257934659719467, 0.04293719679117203, 0.04747294262051582]\n",
      "extrinsic_reward :  2 intrinsic_reward :  413.1221618652344\n",
      "place 70\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 70, 'x_pos_screen': 70, 'y_pos': 99}\n",
      "global_step :  35 , action :  4 action_prob :  [0.05983581021428108, 0.1765412539243698, 0.04411818087100983, 0.5565977096557617, 0.07254990190267563, 0.04290996491909027, 0.0474470891058445]\n",
      "extrinsic_reward :  2 intrinsic_reward :  414.38031005859375\n",
      "place 72\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 72, 'x_pos_screen': 72, 'y_pos': 96}\n",
      "global_step :  36 , action :  3 action_prob :  [0.059858620166778564, 0.17655672132968903, 0.044137999415397644, 0.5564762353897095, 0.07257278263568878, 0.04293042793869972, 0.047467224299907684]\n",
      "extrinsic_reward :  1 intrinsic_reward :  413.4094543457031\n",
      "place 73\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 73, 'x_pos_screen': 73, 'y_pos': 93}\n",
      "global_step :  37 , action :  3 action_prob :  [0.059936508536338806, 0.17660944163799286, 0.04420623555779457, 0.5560587644577026, 0.07265225052833557, 0.04299942031502724, 0.04753729701042175]\n",
      "extrinsic_reward :  2 intrinsic_reward :  413.4603576660156\n",
      "place 75\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 75, 'x_pos_screen': 75, 'y_pos': 89}\n",
      "global_step :  38 , action :  3 action_prob :  [0.059901490807533264, 0.17658491432666779, 0.0441756471991539, 0.5562471747398376, 0.07261674851179123, 0.042967937886714935, 0.04750609025359154]\n",
      "extrinsic_reward :  2 intrinsic_reward :  410.913818359375\n",
      "place 77\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 77, 'x_pos_screen': 77, 'y_pos': 85}\n",
      "global_step :  39 , action :  3 action_prob :  [0.05989561229944229, 0.1765807867050171, 0.04417060688138008, 0.5562785267829895, 0.07261095196008682, 0.042962584644556046, 0.04750099033117294]\n",
      "extrinsic_reward :  1 intrinsic_reward :  408.71636962890625\n",
      "place 79\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 79, 'x_pos_screen': 79, 'y_pos': 79}\n",
      "global_step :  40 , action :  1 action_prob :  [0.05993222817778587, 0.17660561203956604, 0.04420294240117073, 0.5560811161994934, 0.07264889776706696, 0.04299464821815491, 0.047534458339214325]\n",
      "extrinsic_reward :  2 intrinsic_reward :  408.3625183105469\n",
      "place 81\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 81, 'x_pos_screen': 81, 'y_pos': 79}\n",
      "global_step :  41 , action :  0 action_prob :  [0.059912994503974915, 0.1765940636396408, 0.04418640583753586, 0.5561811923980713, 0.07262998074293137, 0.042977653443813324, 0.047517698258161545]\n",
      "extrinsic_reward :  2 intrinsic_reward :  408.1023864746094\n",
      "place 83\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 83, 'x_pos_screen': 82, 'y_pos': 79}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  42 , action :  1 action_prob :  [0.059955403208732605, 0.17662300169467926, 0.04422389715909958, 0.5559524297714233, 0.07267395406961441, 0.043014880269765854, 0.047556452453136444]\n",
      "extrinsic_reward :  1 intrinsic_reward :  397.001708984375\n",
      "place 84\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 84, 'x_pos_screen': 82, 'y_pos': 79}\n",
      "global_step :  43 , action :  3 action_prob :  [0.058751221746206284, 0.1757889837026596, 0.04318161681294441, 0.5623828172683716, 0.0714627355337143, 0.04193944111466408, 0.046493206173181534]\n",
      "extrinsic_reward :  2 intrinsic_reward :  395.2485656738281\n",
      "place 86\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 86, 'x_pos_screen': 83, 'y_pos': 79}\n",
      "global_step :  44 , action :  0 action_prob :  [0.05969994142651558, 0.17644116282463074, 0.04401605948805809, 0.5572709441184998, 0.07244719564914703, 0.042766273021698, 0.047358520328998566]\n",
      "extrinsic_reward :  3 intrinsic_reward :  420.35400390625\n",
      "place 89\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 89, 'x_pos_screen': 84, 'y_pos': 79}\n",
      "global_step :  45 , action :  1 action_prob :  [0.06004399433732033, 0.17667438089847565, 0.04431234672665596, 0.5554479360580444, 0.07278571277856827, 0.043079376220703125, 0.04765627905726433]\n",
      "extrinsic_reward :  2 intrinsic_reward :  425.97314453125\n",
      "place 91\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 91, 'x_pos_screen': 85, 'y_pos': 79}\n",
      "global_step :  46 , action :  3 action_prob :  [0.060306351631879807, 0.1768529862165451, 0.04453866928815842, 0.5540549159049988, 0.07304488122463226, 0.043316490948200226, 0.0478857047855854]\n",
      "extrinsic_reward :  2 intrinsic_reward :  404.64031982421875\n",
      "place 93\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 93, 'x_pos_screen': 86, 'y_pos': 79}\n",
      "global_step :  47 , action :  2 action_prob :  [0.05971347168087959, 0.17644615471363068, 0.04402197524905205, 0.5572296977043152, 0.07244525849819183, 0.042786817997694016, 0.047356728464365005]\n",
      "extrinsic_reward :  2 intrinsic_reward :  417.38751220703125\n",
      "place 95\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 95, 'x_pos_screen': 87, 'y_pos': 79}\n",
      "global_step :  48 , action :  0 action_prob :  [0.06058627739548683, 0.1770295649766922, 0.04478026181459427, 0.5525863170623779, 0.07331907749176025, 0.04356975108385086, 0.04812880977988243]\n",
      "extrinsic_reward :  2 intrinsic_reward :  419.87042236328125\n",
      "place 97\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 97, 'x_pos_screen': 88, 'y_pos': 85}\n",
      "global_step :  49 , action :  3 action_prob :  [0.05949735641479492, 0.17629912495613098, 0.043835751712322235, 0.5583754777908325, 0.0722314864397049, 0.04259270057082176, 0.0471680723130703]\n",
      "extrinsic_reward :  3 intrinsic_reward :  396.69610595703125\n",
      "place 100\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 100, 'x_pos_screen': 89, 'y_pos': 91}\n",
      "global_step :  50 , action :  0 action_prob :  [0.05850725620985031, 0.17560136318206787, 0.04297807440161705, 0.5636804699897766, 0.07123015075922012, 0.04171103239059448, 0.0462917685508728]\n",
      "extrinsic_reward :  2 intrinsic_reward :  421.4447937011719\n",
      "place 102\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 102, 'x_pos_screen': 90, 'y_pos': 96}\n",
      "global_step :  51 , action :  1 action_prob :  [0.05936982110142708, 0.17620787024497986, 0.04373198375105858, 0.559035062789917, 0.07211697101593018, 0.042470112442970276, 0.047068141400814056]\n",
      "extrinsic_reward :  3 intrinsic_reward :  414.65728759765625\n",
      "place 105\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 105, 'x_pos_screen': 91, 'y_pos': 100}\n",
      "global_step :  52 , action :  3 action_prob :  [0.058524295687675476, 0.17561624944210052, 0.04299820959568024, 0.563563883304596, 0.07125931978225708, 0.04172129929065704, 0.04631669074296951]\n",
      "extrinsic_reward :  2 intrinsic_reward :  417.64508056640625\n",
      "place 107\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 107, 'x_pos_screen': 92, 'y_pos': 104}\n",
      "global_step :  53 , action :  6 action_prob :  [0.05913166701793671, 0.17604587972164154, 0.043520789593458176, 0.560323178768158, 0.07186594605445862, 0.04226498678326607, 0.046847499907016754]\n",
      "extrinsic_reward :  3 intrinsic_reward :  422.07135009765625\n",
      "place 110\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 110, 'x_pos_screen': 93, 'y_pos': 106}\n",
      "global_step :  54 , action :  2 action_prob :  [0.059960197657346725, 0.17660681903362274, 0.04424261674284935, 0.5559001564979553, 0.07270382344722748, 0.04300064593553543, 0.047585632652044296]\n",
      "extrinsic_reward :  3 intrinsic_reward :  414.6582946777344\n",
      "place 113\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 113, 'x_pos_screen': 94, 'y_pos': 108}\n",
      "global_step :  55 , action :  3 action_prob :  [0.057870689779520035, 0.17515121400356293, 0.042428452521562576, 0.5670844316482544, 0.0705854520201683, 0.0411505363881588, 0.04572923108935356]\n",
      "extrinsic_reward :  2 intrinsic_reward :  434.4311218261719\n",
      "place 115\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 115, 'x_pos_screen': 95, 'y_pos': 109}\n",
      "global_step :  56 , action :  4 action_prob :  [0.05868121609091759, 0.17572879791259766, 0.043138615787029266, 0.56270432472229, 0.07142727822065353, 0.041855983436107635, 0.04646381363272667]\n",
      "extrinsic_reward :  3 intrinsic_reward :  420.39935302734375\n",
      "place 118\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 118, 'x_pos_screen': 96, 'y_pos': 109}\n",
      "global_step :  57 , action :  3 action_prob :  [0.05786436051130295, 0.17513799667358398, 0.0424313060939312, 0.5670967698097229, 0.07059628516435623, 0.04113384336233139, 0.04573953524231911]\n",
      "extrinsic_reward :  3 intrinsic_reward :  416.499755859375\n",
      "place 121\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 121, 'x_pos_screen': 97, 'y_pos': 108}\n",
      "global_step :  58 , action :  3 action_prob :  [0.058483488857746124, 0.17559006810188293, 0.04296234995126724, 0.5637823343276978, 0.07121561467647552, 0.041687142103910446, 0.046278856694698334]\n",
      "extrinsic_reward :  3 intrinsic_reward :  418.8433837890625\n",
      "place 124\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 124, 'x_pos_screen': 98, 'y_pos': 107}\n",
      "global_step :  59 , action :  2 action_prob :  [0.05811140313744545, 0.17532548308372498, 0.04264277592301369, 0.5657659769058228, 0.070844367146492, 0.04135527089238167, 0.045954737812280655]\n",
      "extrinsic_reward :  2 intrinsic_reward :  425.1942443847656\n",
      "place 127\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 127, 'x_pos_screen': 99, 'y_pos': 104}\n",
      "global_step :  60 , action :  3 action_prob :  [0.05751185491681099, 0.17488344013690948, 0.04213123023509979, 0.5689707398414612, 0.07024776935577393, 0.04081762954592705, 0.045437291264534]\n",
      "extrinsic_reward :  3 intrinsic_reward :  429.1370849609375\n",
      "place 130\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 130, 'x_pos_screen': 100, 'y_pos': 101}\n",
      "global_step :  61 , action :  3 action_prob :  [0.05720612406730652, 0.17465108633041382, 0.04186241701245308, 0.5706449747085571, 0.06992390006780624, 0.04055484011769295, 0.045156631618738174]\n",
      "extrinsic_reward :  3 intrinsic_reward :  424.5062255859375\n",
      "place 133\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 133, 'x_pos_screen': 101, 'y_pos': 97}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  62 , action :  2 action_prob :  [0.05779640004038811, 0.1750834435224533, 0.04236900061368942, 0.5674819350242615, 0.07051991671323776, 0.04107517749071121, 0.04567404091358185]\n",
      "extrinsic_reward :  3 intrinsic_reward :  427.889404296875\n",
      "place 136\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 136, 'x_pos_screen': 102, 'y_pos': 92}\n",
      "global_step :  63 , action :  5 action_prob :  [0.05736677348613739, 0.17476914823055267, 0.04200087860226631, 0.5697803497314453, 0.07008908689022064, 0.04069320857524872, 0.045300524681806564]\n",
      "extrinsic_reward :  3 intrinsic_reward :  425.51751708984375\n",
      "place 139\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 139, 'x_pos_screen': 103, 'y_pos': 87}\n",
      "global_step :  64 , action :  3 action_prob :  [0.05691853538155556, 0.1744239628314972, 0.04162256792187691, 0.5721802115440369, 0.06964630633592606, 0.040289368480443954, 0.04491895064711571]\n",
      "extrinsic_reward :  3 intrinsic_reward :  426.5467224121094\n",
      "place 142\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 142, 'x_pos_screen': 104, 'y_pos': 82}\n",
      "global_step :  65 , action :  2 action_prob :  [0.05658158287405968, 0.1741604506969452, 0.04132755100727081, 0.5740284323692322, 0.06929116696119308, 0.03999768942594528, 0.0446130745112896]\n",
      "extrinsic_reward :  3 intrinsic_reward :  418.9604187011719\n",
      "place 145\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 145, 'x_pos_screen': 105, 'y_pos': 79}\n",
      "global_step :  66 , action :  5 action_prob :  [0.05716670677065849, 0.17460525035858154, 0.04182842746376991, 0.5708786249160767, 0.0698830708861351, 0.04051414504647255, 0.0451238714158535]\n",
      "extrinsic_reward :  3 intrinsic_reward :  424.86810302734375\n",
      "place 148\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 148, 'x_pos_screen': 106, 'y_pos': 85}\n",
      "global_step :  67 , action :  0 action_prob :  [0.056781742721796036, 0.1743154674768448, 0.04150296002626419, 0.5729317665100098, 0.0695028230547905, 0.04016944766044617, 0.044795792549848557]\n",
      "extrinsic_reward :  3 intrinsic_reward :  427.2429504394531\n",
      "place 151\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 151, 'x_pos_screen': 107, 'y_pos': 91}\n",
      "global_step :  68 , action :  1 action_prob :  [0.0562216155230999, 0.17387519776821136, 0.041025787591934204, 0.5759576559066772, 0.06893699616193771, 0.03967277333140373, 0.044309962540864944]\n",
      "extrinsic_reward :  3 intrinsic_reward :  436.3406066894531\n",
      "place 154\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 154, 'x_pos_screen': 108, 'y_pos': 97}\n",
      "global_step :  69 , action :  0 action_prob :  [0.0559970960021019, 0.1737108677625656, 0.040824830532073975, 0.5771909356117249, 0.06869110465049744, 0.039487969130277634, 0.04409714415669441]\n",
      "extrinsic_reward :  3 intrinsic_reward :  435.5131530761719\n",
      "place 157\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 157, 'x_pos_screen': 109, 'y_pos': 102}\n",
      "global_step :  70 , action :  3 action_prob :  [0.056680914014577866, 0.17423461377620697, 0.041412580758333206, 0.5734965205192566, 0.06939081102609634, 0.0400850847363472, 0.04469946771860123]\n",
      "extrinsic_reward :  3 intrinsic_reward :  434.2931823730469\n",
      "place 160\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 160, 'x_pos_screen': 110, 'y_pos': 106}\n",
      "global_step :  71 , action :  3 action_prob :  [0.05626513808965683, 0.17391525208950043, 0.04105988144874573, 0.575726330280304, 0.0689757838845253, 0.03971478343009949, 0.04434297978878021]\n",
      "extrinsic_reward :  3 intrinsic_reward :  439.594970703125\n",
      "place 163\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 163, 'x_pos_screen': 111, 'y_pos': 109}\n",
      "global_step :  72 , action :  1 action_prob :  [0.05572645366191864, 0.173486590385437, 0.04060129076242447, 0.5786415934562683, 0.06842877715826035, 0.03924110531806946, 0.043874166905879974]\n",
      "extrinsic_reward :  3 intrinsic_reward :  440.9208984375\n",
      "place 166\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 166, 'x_pos_screen': 112, 'y_pos': 111}\n",
      "global_step :  73 , action :  3 action_prob :  [0.05568911135196686, 0.1734592616558075, 0.04056519269943237, 0.5788580179214478, 0.06838104873895645, 0.039214957505464554, 0.0438324511051178]\n",
      "extrinsic_reward :  3 intrinsic_reward :  430.8279113769531\n",
      "place 169\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 169, 'x_pos_screen': 112, 'y_pos': 112}\n",
      "global_step :  74 , action :  3 action_prob :  [0.056182410567998886, 0.17385025322437286, 0.04098367318511009, 0.5761967897415161, 0.06887777149677277, 0.039651043713092804, 0.04425802826881409]\n",
      "extrinsic_reward :  3 intrinsic_reward :  433.07281494140625\n",
      "place 172\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 172, 'x_pos_screen': 112, 'y_pos': 113}\n",
      "global_step :  75 , action :  5 action_prob :  [0.05576886236667633, 0.1735314279794693, 0.04063010960817337, 0.5784285664558411, 0.06845495849847794, 0.039290737360715866, 0.04389533773064613]\n",
      "extrinsic_reward :  3 intrinsic_reward :  444.8769226074219\n",
      "place 175\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 175, 'x_pos_screen': 112, 'y_pos': 113}\n",
      "global_step :  76 , action :  4 action_prob :  [0.055137015879154205, 0.17303508520126343, 0.04008744657039642, 0.5818566679954529, 0.06780203431844711, 0.03874609246850014, 0.0433356799185276]\n",
      "extrinsic_reward :  3 intrinsic_reward :  457.9766845703125\n",
      "place 178\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 178, 'x_pos_screen': 112, 'y_pos': 112}\n",
      "global_step :  77 , action :  0 action_prob :  [0.05577735975384712, 0.17354288697242737, 0.04063639044761658, 0.5783807635307312, 0.0684618204832077, 0.039299845695495605, 0.04390092194080353]\n",
      "extrinsic_reward :  3 intrinsic_reward :  449.9039611816406\n",
      "place 181\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 181, 'x_pos_screen': 112, 'y_pos': 110}\n",
      "global_step :  78 , action :  1 action_prob :  [0.05486598238348961, 0.17280447483062744, 0.039863087236881256, 0.5833189487457275, 0.06753553450107574, 0.038501281291246414, 0.04311065003275871]\n",
      "extrinsic_reward :  3 intrinsic_reward :  453.06854248046875\n",
      "place 184\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 184, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  79 , action :  1 action_prob :  [0.05519809573888779, 0.17308929562568665, 0.040138691663742065, 0.5815243721008301, 0.06786125898361206, 0.03880231827497482, 0.04338599368929863]\n",
      "extrinsic_reward :  2 intrinsic_reward :  445.1422119140625\n",
      "place 187\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 187, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  80 , action :  3 action_prob :  [0.05499976500868797, 0.17292910814285278, 0.03997266665101051, 0.5825893878936768, 0.06766436994075775, 0.03862603381276131, 0.04321860894560814]\n",
      "extrinsic_reward :  3 intrinsic_reward :  450.137939453125\n",
      "place 190\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 190, 'x_pos_screen': 112, 'y_pos': 100}\n",
      "global_step :  81 , action :  3 action_prob :  [0.054902829229831696, 0.1728590875864029, 0.03988559916615486, 0.5831231474876404, 0.06755532324314117, 0.038549814373254776, 0.043124228715896606]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "extrinsic_reward :  3 intrinsic_reward :  451.8390808105469\n",
      "place 193\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 193, 'x_pos_screen': 112, 'y_pos': 95}\n",
      "global_step :  82 , action :  3 action_prob :  [0.05506665259599686, 0.17299345135688782, 0.04002031311392784, 0.5822502374649048, 0.06771305203437805, 0.03869832679629326, 0.043257951736450195]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.21044921875\n",
      "place 196\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 196, 'x_pos_screen': 112, 'y_pos': 90}\n",
      "global_step :  83 , action :  3 action_prob :  [0.05493781343102455, 0.17290057241916656, 0.03991219773888588, 0.5829293727874756, 0.06758514046669006, 0.03858642280101776, 0.043148498982191086]\n",
      "extrinsic_reward :  3 intrinsic_reward :  452.8111267089844\n",
      "place 199\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 199, 'x_pos_screen': 112, 'y_pos': 85}\n",
      "global_step :  84 , action :  3 action_prob :  [0.05462449789047241, 0.17264343798160553, 0.039642468094825745, 0.5846461653709412, 0.06725671887397766, 0.03831799700856209, 0.042868856340646744]\n",
      "extrinsic_reward :  3 intrinsic_reward :  454.1781921386719\n",
      "place 202\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 202, 'x_pos_screen': 112, 'y_pos': 80}\n",
      "global_step :  85 , action :  3 action_prob :  [0.05499378964304924, 0.1729448288679123, 0.03995765373110771, 0.5826376080513, 0.06763608008623123, 0.038637686520814896, 0.04319237172603607]\n",
      "extrinsic_reward :  3 intrinsic_reward :  448.388427734375\n",
      "place 205\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 205, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  86 , action :  3 action_prob :  [0.0543215237557888, 0.17238318920135498, 0.039388056844472885, 0.586297333240509, 0.06694970279932022, 0.03805046156048775, 0.0426098108291626]\n",
      "extrinsic_reward :  3 intrinsic_reward :  453.2059631347656\n",
      "place 208\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 208, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  87 , action :  3 action_prob :  [0.05483391880989075, 0.17282217741012573, 0.039815276861190796, 0.583519697189331, 0.06746023893356323, 0.03850780054926872, 0.04304094985127449]\n",
      "extrinsic_reward :  3 intrinsic_reward :  445.8475341796875\n",
      "place 211\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 211, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  88 , action :  3 action_prob :  [0.054487209767103195, 0.1725357621908188, 0.03952312842011452, 0.5853977203369141, 0.06710953265428543, 0.03820247948169708, 0.0427442267537117]\n",
      "extrinsic_reward :  3 intrinsic_reward :  455.3543395996094\n",
      "place 214\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 214, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  89 , action :  3 action_prob :  [0.05484791845083237, 0.17282012104988098, 0.039828043431043625, 0.5834580063819885, 0.06747496873140335, 0.038515813648700714, 0.04305516928434372]\n",
      "extrinsic_reward :  3 intrinsic_reward :  445.5445861816406\n",
      "place 217\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 217, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  90 , action :  4 action_prob :  [0.05507702752947807, 0.17301683127880096, 0.04001585766673088, 0.5822265148162842, 0.06769665330648422, 0.038724955171346664, 0.04324224591255188]\n",
      "extrinsic_reward :  3 intrinsic_reward :  449.572998046875\n",
      "place 220\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 220, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  91 , action :  3 action_prob :  [0.054978348314762115, 0.17293889820575714, 0.039937712252140045, 0.582737386226654, 0.06760803610086441, 0.03863224759697914, 0.04316739737987518]\n",
      "extrinsic_reward :  3 intrinsic_reward :  451.8525695800781\n",
      "place 223\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 223, 'x_pos_screen': 112, 'y_pos': 85}\n",
      "global_step :  92 , action :  1 action_prob :  [0.05503034591674805, 0.17296336591243744, 0.03998290002346039, 0.5824740529060364, 0.06766173243522644, 0.03867286071181297, 0.043214745819568634]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.03875732421875\n",
      "place 226\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 226, 'x_pos_screen': 112, 'y_pos': 91}\n",
      "global_step :  93 , action :  4 action_prob :  [0.0552925206720829, 0.17317920923233032, 0.04020748659968376, 0.5810409784317017, 0.06793385744094849, 0.03889881819486618, 0.04344717040657997]\n",
      "extrinsic_reward :  3 intrinsic_reward :  450.20208740234375\n",
      "place 229\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 229, 'x_pos_screen': 112, 'y_pos': 96}\n",
      "global_step :  94 , action :  1 action_prob :  [0.054656900465488434, 0.17265667021274567, 0.03966560959815979, 0.5845023989677429, 0.06728129833936691, 0.038345277309417725, 0.042891889810562134]\n",
      "extrinsic_reward :  3 intrinsic_reward :  454.86065673828125\n",
      "place 232\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 232, 'x_pos_screen': 112, 'y_pos': 100}\n",
      "global_step :  95 , action :  1 action_prob :  [0.055429279804229736, 0.17327472567558289, 0.040323786437511444, 0.5803162455558777, 0.06807376444339752, 0.039014726877212524, 0.04356753081083298]\n",
      "extrinsic_reward :  3 intrinsic_reward :  449.5682067871094\n",
      "place 235\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 235, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  96 , action :  3 action_prob :  [0.055175188928842545, 0.17307069897651672, 0.04011310264468193, 0.5816716551780701, 0.06782595813274384, 0.038786932826042175, 0.04335646331310272]\n",
      "extrinsic_reward :  3 intrinsic_reward :  450.96173095703125\n",
      "place 238\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 238, 'x_pos_screen': 112, 'y_pos': 106}\n",
      "global_step :  97 , action :  1 action_prob :  [0.05555744096636772, 0.17336176335811615, 0.04043720290064812, 0.5796231627464294, 0.06821388751268387, 0.03911780193448067, 0.043688736855983734]\n",
      "extrinsic_reward :  3 intrinsic_reward :  446.22076416015625\n",
      "place 241\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 241, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  98 , action :  3 action_prob :  [0.05594741553068161, 0.17366419732570648, 0.04076893627643585, 0.5775246620178223, 0.0686083734035492, 0.03945981338620186, 0.04402657598257065]\n",
      "extrinsic_reward :  3 intrinsic_reward :  449.56390380859375\n",
      "place 244\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 244, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  99 , action :  2 action_prob :  [0.05560985207557678, 0.1734103560447693, 0.040485039353370667, 0.5793178081512451, 0.06827490031719208, 0.039161451160907745, 0.04374048113822937]\n",
      "extrinsic_reward :  2 intrinsic_reward :  453.9360656738281\n",
      "place 247\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 247, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  100 , action :  1 action_prob :  [0.05580276623368263, 0.17353558540344238, 0.040651094168424606, 0.5783016085624695, 0.06847424060106277, 0.03932090476155281, 0.04391385614871979]\n",
      "extrinsic_reward :  3 intrinsic_reward :  455.9727783203125\n",
      "place 250\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 250, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  101 , action :  1 action_prob :  [0.055850327014923096, 0.1735943704843521, 0.04069139063358307, 0.5780189633369446, 0.06852340698242188, 0.039367035031318665, 0.04395455867052078]\n",
      "extrinsic_reward :  3 intrinsic_reward :  451.303466796875\n",
      "place 253\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 253, 'x_pos_screen': 112, 'y_pos': 107}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  102 , action :  3 action_prob :  [0.05519409477710724, 0.1730620563030243, 0.040130969136953354, 0.5815882682800293, 0.06785078346729279, 0.03879405930638313, 0.043379660695791245]\n",
      "extrinsic_reward :  3 intrinsic_reward :  439.0389404296875\n",
      "place 256\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 256, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  103 , action :  2 action_prob :  [0.05574695020914078, 0.1735055148601532, 0.040597692131996155, 0.5786048173904419, 0.06840935349464417, 0.03927904739975929, 0.0438566729426384]\n",
      "extrinsic_reward :  3 intrinsic_reward :  443.80694580078125\n",
      "place 259\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 259, 'x_pos_screen': 112, 'y_pos': 101}\n",
      "global_step :  104 , action :  1 action_prob :  [0.05549899488687515, 0.17330823838710785, 0.04038805514574051, 0.5799421072006226, 0.06815885007381439, 0.03906135633587837, 0.04364238306879997]\n",
      "extrinsic_reward :  3 intrinsic_reward :  451.0122375488281\n",
      "place 262\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 262, 'x_pos_screen': 112, 'y_pos': 97}\n",
      "global_step :  105 , action :  3 action_prob :  [0.05558806285262108, 0.17337508499622345, 0.04046134650707245, 0.579474925994873, 0.06824416667222977, 0.03914080932736397, 0.04371559992432594]\n",
      "extrinsic_reward :  3 intrinsic_reward :  444.8632507324219\n",
      "place 265\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 265, 'x_pos_screen': 112, 'y_pos': 92}\n",
      "global_step :  106 , action :  3 action_prob :  [0.055864978581666946, 0.1735949069261551, 0.04069189354777336, 0.5779960751533508, 0.0685158222913742, 0.039389219135046005, 0.04394712671637535]\n",
      "extrinsic_reward :  3 intrinsic_reward :  440.51348876953125\n",
      "place 268\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 268, 'x_pos_screen': 112, 'y_pos': 87}\n",
      "global_step :  107 , action :  3 action_prob :  [0.055495087057352066, 0.17330215871334076, 0.04038267955183983, 0.5799745321273804, 0.06815095990896225, 0.03905894607305527, 0.04363555088639259]\n",
      "extrinsic_reward :  3 intrinsic_reward :  456.91558837890625\n",
      "place 271\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 271, 'x_pos_screen': 112, 'y_pos': 82}\n",
      "global_step :  108 , action :  0 action_prob :  [0.055299222469329834, 0.17314717173576355, 0.040210988372564316, 0.5810534954071045, 0.06793946772813797, 0.03889577090740204, 0.04345396161079407]\n",
      "extrinsic_reward :  3 intrinsic_reward :  448.1742248535156\n",
      "place 274\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 274, 'x_pos_screen': 112, 'y_pos': 77}\n",
      "global_step :  109 , action :  1 action_prob :  [0.05547052249312401, 0.17328447103500366, 0.0403597317636013, 0.5801147818565369, 0.06811995059251785, 0.03904251381754875, 0.043607961386442184]\n",
      "extrinsic_reward :  2 intrinsic_reward :  445.2236022949219\n",
      "place 276\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 276, 'x_pos_screen': 111, 'y_pos': 72}\n",
      "global_step :  110 , action :  3 action_prob :  [0.05483851581811905, 0.17277564108371735, 0.039820387959480286, 0.5835483074188232, 0.06746868044137955, 0.03849644213914871, 0.043052028864622116]\n",
      "extrinsic_reward :  0 intrinsic_reward :  440.7919006347656\n",
      "place 276\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 276, 'x_pos_screen': 111, 'y_pos': 67}\n",
      "global_step :  111 , action :  6 action_prob :  [0.05501655116677284, 0.17293381690979004, 0.03996647149324417, 0.5825875401496887, 0.06763871759176254, 0.03866163641214371, 0.04319527745246887]\n",
      "extrinsic_reward :  0 intrinsic_reward :  441.58648681640625\n",
      "place 276\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 276, 'x_pos_screen': 111, 'y_pos': 62}\n",
      "global_step :  112 , action :  1 action_prob :  [0.05502002313733101, 0.17294026911258698, 0.03996839001774788, 0.5825680494308472, 0.0676402747631073, 0.038666922599077225, 0.04319603741168976]\n",
      "extrinsic_reward :  0 intrinsic_reward :  442.69049072265625\n",
      "place 276\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 276, 'x_pos_screen': 111, 'y_pos': 57}\n",
      "global_step :  113 , action :  3 action_prob :  [0.055022820830345154, 0.17294518649578094, 0.03996992111206055, 0.5825528502464294, 0.06764143705368042, 0.03867122530937195, 0.04319657012820244]\n",
      "extrinsic_reward :  0 intrinsic_reward :  443.34619140625\n",
      "place 276\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 276, 'x_pos_screen': 111, 'y_pos': 52}\n",
      "global_step :  114 , action :  6 action_prob :  [0.055011022835969925, 0.17293484508991241, 0.039959609508514404, 0.5826184749603271, 0.06762923300266266, 0.03866054490208626, 0.043186306953430176]\n",
      "extrinsic_reward :  0 intrinsic_reward :  442.58099365234375\n",
      "place 276\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 276, 'x_pos_screen': 111, 'y_pos': 47}\n",
      "global_step :  115 , action :  3 action_prob :  [0.05501490831375122, 0.17293761670589447, 0.03996279835700989, 0.5825983285903931, 0.06763291358947754, 0.03866410255432129, 0.04318937659263611]\n",
      "extrinsic_reward :  1 intrinsic_reward :  442.85906982421875\n",
      "place 277\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 277, 'x_pos_screen': 112, 'y_pos': 42}\n",
      "global_step :  116 , action :  1 action_prob :  [0.054989684373140335, 0.1729167103767395, 0.03994122892618179, 0.5827358365058899, 0.06760726869106293, 0.03864153102040291, 0.04316779598593712]\n",
      "extrinsic_reward :  0 intrinsic_reward :  442.92315673828125\n",
      "place 277\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 277, 'x_pos_screen': 112, 'y_pos': 37}\n",
      "global_step :  117 , action :  4 action_prob :  [0.05497963726520538, 0.172908216714859, 0.03993258997797966, 0.5827907919883728, 0.06759701669216156, 0.03863250091671944, 0.043159160763025284]\n",
      "extrinsic_reward :  0 intrinsic_reward :  442.7105407714844\n",
      "place 277\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 277, 'x_pos_screen': 112, 'y_pos': 32}\n",
      "global_step :  118 , action :  0 action_prob :  [0.05497755855321884, 0.17290645837783813, 0.03993088752031326, 0.5828019380569458, 0.06759504973888397, 0.03863053023815155, 0.04315754398703575]\n",
      "extrinsic_reward :  0 intrinsic_reward :  442.8580322265625\n",
      "place 277\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 277, 'x_pos_screen': 112, 'y_pos': 27}\n",
      "global_step :  119 , action :  3 action_prob :  [0.05497782677412033, 0.17290669679641724, 0.03993111103773117, 0.5828006267547607, 0.06759533286094666, 0.03863076493144035, 0.043157778680324554]\n",
      "extrinsic_reward :  0 intrinsic_reward :  442.7744445800781\n",
      "place 278\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 278, 'x_pos_screen': 113, 'y_pos': 22}\n",
      "global_step :  120 , action :  3 action_prob :  [0.05497698485851288, 0.1729060262441635, 0.03993043676018715, 0.5828049182891846, 0.06759454309940338, 0.03862999752163887, 0.04315710440278053]\n",
      "extrinsic_reward :  0 intrinsic_reward :  442.7475891113281\n",
      "place 278\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 278, 'x_pos_screen': 113, 'y_pos': 17}\n",
      "global_step :  121 , action :  3 action_prob :  [0.05496697500348091, 0.17289823293685913, 0.039921835064888, 0.5828591585159302, 0.06758418679237366, 0.03862140327692032, 0.04314827173948288]\n",
      "extrinsic_reward :  1 intrinsic_reward :  442.5455627441406\n",
      "place 279\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 279, 'x_pos_screen': 114, 'y_pos': 12}\n",
      "global_step :  122 , action :  1 action_prob :  [0.05496916547417641, 0.17289993166923523, 0.0399235300719738, 0.5828480124473572, 0.0675860196352005, 0.03862355276942253, 0.04314980283379555]\n",
      "extrinsic_reward :  1 intrinsic_reward :  442.68414306640625\n",
      "place 280\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 280, 'x_pos_screen': 115, 'y_pos': 7}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  123 , action :  4 action_prob :  [0.05496842786669731, 0.17289933562278748, 0.03992292657494545, 0.5828518867492676, 0.06758532673120499, 0.038622863590717316, 0.04314921423792839]\n",
      "extrinsic_reward :  0 intrinsic_reward :  442.68414306640625\n",
      "place 280\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 280, 'x_pos_screen': 115, 'y_pos': 2}\n",
      "epoch :  1 , global_step :  124\n",
      "global_step :  1 , action :  3 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.56402587890625\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  2 , action :  1 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.56402587890625\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  3 , action :  2 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  410.9014892578125\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  4 , action :  3 action_prob :  [0.05981843173503876, 0.17652776837348938, 0.0441044345498085, 0.5566869974136353, 0.07253562659025192, 0.04289211705327034, 0.047434646636247635]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.946533203125\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 84}\n",
      "global_step :  5 , action :  5 action_prob :  [0.05979273095726967, 0.1765090972185135, 0.04408237338066101, 0.5568243265151978, 0.0725105032324791, 0.04286839812994003, 0.04741258919239044]\n",
      "extrinsic_reward :  0 intrinsic_reward :  413.3307800292969\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 89}\n",
      "global_step :  6 , action :  0 action_prob :  [0.05978989228606224, 0.1765068620443344, 0.04407934099435806, 0.5568422079086304, 0.07250624150037766, 0.04286649823188782, 0.04740886762738228]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.8639221191406\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 93}\n",
      "global_step :  7 , action :  3 action_prob :  [0.05976428836584091, 0.17649048566818237, 0.044056519865989685, 0.5569798350334167, 0.0724792331457138, 0.04284464567899704, 0.04738491773605347]\n",
      "extrinsic_reward :  0 intrinsic_reward :  410.5800476074219\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 96}\n",
      "global_step :  8 , action :  6 action_prob :  [0.05967511236667633, 0.17643098533153534, 0.043978311121463776, 0.5574571490287781, 0.07238789647817612, 0.042766306549310684, 0.047304295003414154]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.41595458984375\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 99}\n",
      "global_step :  9 , action :  3 action_prob :  [0.05970977991819382, 0.1764560043811798, 0.04400821402668953, 0.5572711229324341, 0.07242247462272644, 0.04279779642820358, 0.04733460023999214]\n",
      "extrinsic_reward :  1 intrinsic_reward :  413.4479675292969\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 101}\n",
      "global_step :  10 , action :  3 action_prob :  [0.05966973304748535, 0.17642910778522491, 0.04397285729646683, 0.5574865341186523, 0.07238102704286575, 0.04276272654533386, 0.04729806259274483]\n",
      "extrinsic_reward :  0 intrinsic_reward :  413.80108642578125\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 102}\n",
      "global_step :  11 , action :  1 action_prob :  [0.059666890650987625, 0.17642684280872345, 0.0439702607691288, 0.5575026273727417, 0.07237780094146729, 0.04276029393076897, 0.04729529097676277]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.778076171875\n",
      "place 43\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 43, 'x_pos_screen': 43, 'y_pos': 103}\n",
      "global_step :  12 , action :  3 action_prob :  [0.05966455116868019, 0.17642518877983093, 0.04396805539727211, 0.5575158596038818, 0.0723750963807106, 0.04275839403271675, 0.047292884439229965]\n",
      "extrinsic_reward :  0 intrinsic_reward :  413.63372802734375\n",
      "place 43\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 43, 'x_pos_screen': 43, 'y_pos': 103}\n",
      "global_step :  13 , action :  4 action_prob :  [0.05973539501428604, 0.17647381126880646, 0.04402998462319374, 0.5571358799934387, 0.07244718819856644, 0.04282139614224434, 0.04735635593533516]\n",
      "extrinsic_reward :  1 intrinsic_reward :  413.63372802734375\n",
      "place 44\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 44, 'x_pos_screen': 44, 'y_pos': 102}\n",
      "global_step :  14 , action :  3 action_prob :  [0.05973539501428604, 0.17647381126880646, 0.04402998462319374, 0.5571358799934387, 0.07244718819856644, 0.04282139614224434, 0.04735635593533516]\n",
      "extrinsic_reward :  0 intrinsic_reward :  410.5721435546875\n",
      "place 44\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 44, 'x_pos_screen': 44, 'y_pos': 101}\n",
      "global_step :  15 , action :  3 action_prob :  [0.0597432442009449, 0.17647893726825714, 0.04403693974018097, 0.5570937395095825, 0.07245539128780365, 0.04282813146710396, 0.04736362397670746]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.04901123046875\n",
      "place 45\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 45, 'x_pos_screen': 45, 'y_pos': 99}\n",
      "global_step :  16 , action :  3 action_prob :  [0.05974024161696434, 0.1764770746231079, 0.04403435066342354, 0.5571093559265137, 0.07245247811079025, 0.042825400829315186, 0.047361064702272415]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.744140625\n",
      "place 46\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 46, 'x_pos_screen': 46, 'y_pos': 96}\n",
      "global_step :  17 , action :  3 action_prob :  [0.05975423380732536, 0.17648623883724213, 0.044046930968761444, 0.5570336580276489, 0.07246733456850052, 0.04283739626407623, 0.04737420007586479]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.4855041503906\n",
      "place 46\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 46, 'x_pos_screen': 46, 'y_pos': 93}\n",
      "global_step :  18 , action :  3 action_prob :  [0.05979013070464134, 0.1765107959508896, 0.044078655540943146, 0.5568400621414185, 0.07250454276800156, 0.04286894574761391, 0.04740697890520096]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.37548828125\n",
      "place 47\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 47, 'x_pos_screen': 47, 'y_pos': 89}\n",
      "global_step :  19 , action :  3 action_prob :  [0.0597451888024807, 0.17647996544837952, 0.044039320200681686, 0.5570812225341797, 0.07245869934558868, 0.04282909631729126, 0.04736656695604324]\n",
      "extrinsic_reward :  0 intrinsic_reward :  410.4362487792969\n",
      "place 48\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 48, 'x_pos_screen': 48, 'y_pos': 85}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  20 , action :  3 action_prob :  [0.059812504798173904, 0.17652513086795807, 0.04409845545887947, 0.556720495223999, 0.07252772897481918, 0.04288829118013382, 0.04742753505706787]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.7789611816406\n",
      "place 49\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 49, 'x_pos_screen': 49, 'y_pos': 79}\n",
      "global_step :  21 , action :  3 action_prob :  [0.059780243784189224, 0.1765032708644867, 0.04407006874680519, 0.5568938851356506, 0.07249433547258377, 0.04286014288663864, 0.04739804565906525]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.1164245605469\n",
      "place 50\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 50, 'x_pos_screen': 50, 'y_pos': 79}\n",
      "global_step :  22 , action :  3 action_prob :  [0.05984208360314369, 0.1765448898077011, 0.04412464797496796, 0.5565611720085144, 0.07255835086107254, 0.04291433095932007, 0.04745452478528023]\n",
      "extrinsic_reward :  2 intrinsic_reward :  409.18017578125\n",
      "place 52\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 52, 'x_pos_screen': 52, 'y_pos': 79}\n",
      "global_step :  23 , action :  1 action_prob :  [0.059825703501701355, 0.17653417587280273, 0.04411044716835022, 0.5566479563713074, 0.07254190742969513, 0.04289987310767174, 0.04743995890021324]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.3902893066406\n",
      "place 53\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 53, 'x_pos_screen': 53, 'y_pos': 79}\n",
      "global_step :  24 , action :  3 action_prob :  [0.05985331907868385, 0.17655277252197266, 0.04413452371954918, 0.5565003752708435, 0.072569839656353, 0.042924366891384125, 0.047464653849601746]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.6632995605469\n",
      "place 54\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 54, 'x_pos_screen': 54, 'y_pos': 79}\n",
      "global_step :  25 , action :  3 action_prob :  [0.05987146869301796, 0.17656506597995758, 0.044150177389383316, 0.5564041137695312, 0.07258796691894531, 0.042940545827150345, 0.047480661422014236]\n",
      "extrinsic_reward :  2 intrinsic_reward :  409.59912109375\n",
      "place 56\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 56, 'x_pos_screen': 56, 'y_pos': 79}\n",
      "global_step :  26 , action :  5 action_prob :  [0.05981588736176491, 0.1765275001525879, 0.04410170391201973, 0.5567012429237366, 0.07253146916627884, 0.04289143532514572, 0.047430746257305145]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.1701354980469\n",
      "place 57\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 57, 'x_pos_screen': 57, 'y_pos': 79}\n",
      "global_step :  27 , action :  3 action_prob :  [0.059843238443136215, 0.17654627561569214, 0.04412558302283287, 0.5565545558929443, 0.07255936414003372, 0.04291563108563423, 0.04745536670088768]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.5946044921875\n",
      "place 59\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 59, 'x_pos_screen': 59, 'y_pos': 84}\n",
      "global_step :  28 , action :  3 action_prob :  [0.05982102453708649, 0.17653118073940277, 0.04410625994205475, 0.5566732883453369, 0.07253687083721161, 0.04289591684937477, 0.04743552207946777]\n",
      "extrinsic_reward :  2 intrinsic_reward :  410.733642578125\n",
      "place 61\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 61, 'x_pos_screen': 61, 'y_pos': 89}\n",
      "global_step :  29 , action :  3 action_prob :  [0.05981152132153511, 0.17652453482151031, 0.044097594916820526, 0.5567255616188049, 0.0725264772772789, 0.04288776218891144, 0.04742639511823654]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.50830078125\n",
      "place 62\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 62, 'x_pos_screen': 62, 'y_pos': 93}\n",
      "global_step :  30 , action :  3 action_prob :  [0.059895191341638565, 0.17658203840255737, 0.04417043551802635, 0.5562779307365417, 0.07261104881763458, 0.042962443083524704, 0.04750092700123787]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.6245422363281\n",
      "place 64\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 64, 'x_pos_screen': 64, 'y_pos': 96}\n",
      "global_step :  31 , action :  3 action_prob :  [0.059822358191013336, 0.17653274536132812, 0.04410634934902191, 0.5566692352294922, 0.07253614813089371, 0.04289825260639191, 0.04743487015366554]\n",
      "extrinsic_reward :  2 intrinsic_reward :  413.92236328125\n",
      "place 66\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 66, 'x_pos_screen': 66, 'y_pos': 99}\n",
      "global_step :  32 , action :  1 action_prob :  [0.059851955622434616, 0.17655207216739655, 0.044132813811302185, 0.5565095543861389, 0.07256738841533661, 0.042923714965581894, 0.04746248200535774]\n",
      "extrinsic_reward :  2 intrinsic_reward :  413.6008605957031\n",
      "place 68\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 68, 'x_pos_screen': 68, 'y_pos': 102}\n",
      "global_step :  33 , action :  1 action_prob :  [0.05980488285422325, 0.17652001976966858, 0.04409138485789299, 0.5567628741264343, 0.07251890003681183, 0.04288221150636673, 0.04741977900266647]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.75787353515625\n",
      "place 69\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 69, 'x_pos_screen': 69, 'y_pos': 104}\n",
      "global_step :  34 , action :  3 action_prob :  [0.05986862629652023, 0.17656423151493073, 0.04414660856127739, 0.5564221143722534, 0.07258281856775284, 0.04293953627347946, 0.047476015985012054]\n",
      "extrinsic_reward :  2 intrinsic_reward :  414.2604675292969\n",
      "place 71\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 71, 'x_pos_screen': 71, 'y_pos': 105}\n",
      "global_step :  35 , action :  1 action_prob :  [0.05989668890833855, 0.17658324539661407, 0.04417122155427933, 0.5562716722488403, 0.07261144369840622, 0.04296443983912468, 0.047501254826784134]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.4241943359375\n",
      "place 73\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 73, 'x_pos_screen': 73, 'y_pos': 106}\n",
      "global_step :  36 , action :  4 action_prob :  [0.05985258147120476, 0.17655299603939056, 0.04413269832730293, 0.5565082430839539, 0.07256656140089035, 0.04292525723576546, 0.047461703419685364]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.9230041503906\n",
      "place 75\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 75, 'x_pos_screen': 75, 'y_pos': 106}\n",
      "global_step :  37 , action :  2 action_prob :  [0.059888120740652084, 0.17657625675201416, 0.044164396822452545, 0.5563167333602905, 0.07260380685329437, 0.04295601695775986, 0.047494661062955856]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.0286560058594\n",
      "place 76\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 76, 'x_pos_screen': 76, 'y_pos': 105}\n",
      "global_step :  38 , action :  3 action_prob :  [0.05990978702902794, 0.17659102380275726, 0.04418322443962097, 0.5562011003494263, 0.07262559980154037, 0.04297542944550514, 0.047513868659734726]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.3691711425781\n",
      "place 78\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 78, 'x_pos_screen': 78, 'y_pos': 104}\n",
      "global_step :  39 , action :  1 action_prob :  [0.05996954068541527, 0.17663192749023438, 0.04423505812883377, 0.5558823347091675, 0.07268532365560532, 0.04302927106618881, 0.0475664883852005]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.7585144042969\n",
      "place 80\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 80, 'x_pos_screen': 80, 'y_pos': 103}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  40 , action :  3 action_prob :  [0.059946198016405106, 0.17661651968955994, 0.04421427845954895, 0.5560083985328674, 0.07266070693731308, 0.043009210377931595, 0.047544728964567184]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.8077697753906\n",
      "place 82\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 82, 'x_pos_screen': 81, 'y_pos': 100}\n",
      "global_step :  41 , action :  1 action_prob :  [0.05998970940709114, 0.17664526402950287, 0.04425302520394325, 0.5557738542556763, 0.0727061852812767, 0.043046947568655014, 0.047585006803274155]\n",
      "extrinsic_reward :  1 intrinsic_reward :  401.0556640625\n",
      "place 83\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 83, 'x_pos_screen': 81, 'y_pos': 98}\n",
      "global_step :  42 , action :  1 action_prob :  [0.05879192799329758, 0.17581626772880554, 0.04321670904755592, 0.5621675252914429, 0.07150309532880783, 0.04197574406862259, 0.04652872681617737]\n",
      "extrinsic_reward :  2 intrinsic_reward :  399.0923156738281\n",
      "place 85\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 85, 'x_pos_screen': 82, 'y_pos': 95}\n",
      "global_step :  43 , action :  3 action_prob :  [0.05964326113462448, 0.17640012502670288, 0.043966684490442276, 0.5575773119926453, 0.07238910347223282, 0.04271583631634712, 0.04730764776468277]\n",
      "extrinsic_reward :  2 intrinsic_reward :  422.0478820800781\n",
      "place 87\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 87, 'x_pos_screen': 83, 'y_pos': 91}\n",
      "global_step :  44 , action :  1 action_prob :  [0.06015656888484955, 0.17674650251865387, 0.04441085457801819, 0.5548508167266846, 0.07289941608905792, 0.04317881911993027, 0.047757118940353394]\n",
      "extrinsic_reward :  2 intrinsic_reward :  408.1667175292969\n",
      "place 89\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 89, 'x_pos_screen': 84, 'y_pos': 86}\n",
      "global_step :  45 , action :  1 action_prob :  [0.059021420776844025, 0.17596441507339478, 0.043424706906080246, 0.5609187483787537, 0.07175479084253311, 0.04216562584042549, 0.046750206500291824]\n",
      "extrinsic_reward :  1 intrinsic_reward :  423.6396484375\n",
      "place 90\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 90, 'x_pos_screen': 84, 'y_pos': 81}\n",
      "global_step :  46 , action :  3 action_prob :  [0.06026441231369972, 0.176823690533638, 0.04450169950723648, 0.5542817711830139, 0.07300178706645966, 0.04327896982431412, 0.047847773879766464]\n",
      "extrinsic_reward :  2 intrinsic_reward :  405.64892578125\n",
      "place 92\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 92, 'x_pos_screen': 85, 'y_pos': 79}\n",
      "global_step :  47 , action :  3 action_prob :  [0.059787120670080185, 0.1764962524175644, 0.04408620297908783, 0.5568355321884155, 0.07252008467912674, 0.0428520031273365, 0.04742273688316345]\n",
      "extrinsic_reward :  2 intrinsic_reward :  418.2100830078125\n",
      "place 94\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 94, 'x_pos_screen': 86, 'y_pos': 79}\n",
      "global_step :  48 , action :  6 action_prob :  [0.060642585158348083, 0.17706723511219025, 0.04482938349246979, 0.5522856712341309, 0.07337585091590881, 0.04362011328339577, 0.04817914962768555]\n",
      "extrinsic_reward :  2 intrinsic_reward :  419.09417724609375\n",
      "place 96\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 96, 'x_pos_screen': 87, 'y_pos': 79}\n",
      "global_step :  49 , action :  0 action_prob :  [0.0595015212893486, 0.17630206048488617, 0.04384007304906845, 0.5583508014678955, 0.07223691046237946, 0.04259595647454262, 0.04717276617884636]\n",
      "extrinsic_reward :  2 intrinsic_reward :  397.8135070800781\n",
      "place 98\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 98, 'x_pos_screen': 88, 'y_pos': 79}\n",
      "global_step :  50 , action :  1 action_prob :  [0.058409564197063446, 0.17553287744522095, 0.04289356619119644, 0.5642029047012329, 0.07113087922334671, 0.04162545129656792, 0.0462048202753067]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.1968688964844\n",
      "place 100\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 100, 'x_pos_screen': 89, 'y_pos': 79}\n",
      "global_step :  51 , action :  4 action_prob :  [0.05912540480494499, 0.1760357916355133, 0.043524276465177536, 0.5603277087211609, 0.07188034057617188, 0.04224532097578049, 0.04686126485466957]\n",
      "extrinsic_reward :  2 intrinsic_reward :  422.72998046875\n",
      "place 102\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 102, 'x_pos_screen': 90, 'y_pos': 79}\n",
      "global_step :  52 , action :  3 action_prob :  [0.05934781953692436, 0.1761939376592636, 0.04371316358447075, 0.559149980545044, 0.0720953568816185, 0.04245070368051529, 0.04704896733164787]\n",
      "extrinsic_reward :  2 intrinsic_reward :  418.90545654296875\n",
      "place 104\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 104, 'x_pos_screen': 91, 'y_pos': 85}\n",
      "global_step :  53 , action :  3 action_prob :  [0.05846310034394264, 0.17557503283023834, 0.04294457659125328, 0.5638907551765442, 0.07119691371917725, 0.041667673736810684, 0.04626189544796944]\n",
      "extrinsic_reward :  2 intrinsic_reward :  426.2073059082031\n",
      "place 106\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 106, 'x_pos_screen': 92, 'y_pos': 91}\n",
      "global_step :  54 , action :  3 action_prob :  [0.05959568917751312, 0.17636580765247345, 0.043927840888500214, 0.5578271746635437, 0.07234352082014084, 0.04267207905650139, 0.04726789519190788]\n",
      "extrinsic_reward :  2 intrinsic_reward :  417.3808288574219\n",
      "place 108\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 108, 'x_pos_screen': 93, 'y_pos': 96}\n",
      "global_step :  55 , action :  4 action_prob :  [0.05908899009227753, 0.17601382732391357, 0.04348405450582504, 0.560553789138794, 0.07182289659976959, 0.042226359248161316, 0.04681004583835602]\n",
      "extrinsic_reward :  3 intrinsic_reward :  422.2034912109375\n",
      "place 111\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 111, 'x_pos_screen': 94, 'y_pos': 100}\n",
      "global_step :  56 , action :  3 action_prob :  [0.059995707124471664, 0.17663051187992096, 0.04427334666252136, 0.5557116270065308, 0.0727396160364151, 0.043031927198171616, 0.04761723801493645]\n",
      "extrinsic_reward :  2 intrinsic_reward :  416.8113098144531\n",
      "place 113\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 113, 'x_pos_screen': 95, 'y_pos': 104}\n",
      "global_step :  57 , action :  0 action_prob :  [0.058039698749780655, 0.17526663839817047, 0.04257677495479584, 0.5661778450012207, 0.07076060771942139, 0.041296057403087616, 0.045882318168878555]\n",
      "extrinsic_reward :  3 intrinsic_reward :  413.1119384765625\n",
      "place 116\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 116, 'x_pos_screen': 96, 'y_pos': 106}\n",
      "global_step :  58 , action :  4 action_prob :  [0.05827982723712921, 0.1754414141178131, 0.04279323294758797, 0.5648495554924011, 0.07102589309215546, 0.04149641469120979, 0.04611370339989662]\n",
      "extrinsic_reward :  2 intrinsic_reward :  421.40301513671875\n",
      "place 118\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 118, 'x_pos_screen': 97, 'y_pos': 108}\n",
      "global_step :  59 , action :  5 action_prob :  [0.05787292867898941, 0.17514297366142273, 0.042438752949237823, 0.5670521259307861, 0.07060505449771881, 0.04114088416099548, 0.045747309923172]\n",
      "extrinsic_reward :  2 intrinsic_reward :  431.44207763671875\n",
      "place 121\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 121, 'x_pos_screen': 98, 'y_pos': 109}\n",
      "global_step :  60 , action :  1 action_prob :  [0.0588376484811306, 0.17584477365016937, 0.04327123239636421, 0.5618703961372375, 0.0715801864862442, 0.04199804365634918, 0.04659770801663399]\n",
      "extrinsic_reward :  3 intrinsic_reward :  425.92822265625\n",
      "place 124\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 124, 'x_pos_screen': 99, 'y_pos': 109}\n",
      "global_step :  61 , action :  3 action_prob :  [0.059098899364471436, 0.17602869868278503, 0.043493520468473434, 0.5604872703552246, 0.07183492928743362, 0.04223652184009552, 0.04682014510035515]\n",
      "extrinsic_reward :  3 intrinsic_reward :  409.5137634277344\n",
      "place 127\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 127, 'x_pos_screen': 100, 'y_pos': 108}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  62 , action :  4 action_prob :  [0.057391319423913956, 0.17479312419891357, 0.04201708734035492, 0.56966233253479, 0.07010151445865631, 0.040724508464336395, 0.04531010612845421]\n",
      "extrinsic_reward :  2 intrinsic_reward :  434.8276672363281\n",
      "place 129\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 129, 'x_pos_screen': 101, 'y_pos': 107}\n",
      "global_step :  63 , action :  3 action_prob :  [0.05783672630786896, 0.17512246966362, 0.04240734130144119, 0.5672383904457092, 0.07056965678930283, 0.04110906645655632, 0.04571628198027611]\n",
      "extrinsic_reward :  3 intrinsic_reward :  429.40606689453125\n",
      "place 132\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 132, 'x_pos_screen': 102, 'y_pos': 104}\n",
      "global_step :  64 , action :  3 action_prob :  [0.05716262012720108, 0.17462217807769775, 0.041823599487543106, 0.5708794593811035, 0.06987745314836502, 0.040518734604120255, 0.04511594399809837]\n",
      "extrinsic_reward :  3 intrinsic_reward :  424.01397705078125\n",
      "place 135\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 135, 'x_pos_screen': 103, 'y_pos': 101}\n",
      "global_step :  65 , action :  3 action_prob :  [0.057817596942186356, 0.17510037124156952, 0.04238768294453621, 0.5673649311065674, 0.07054197788238525, 0.04109435901045799, 0.045693036168813705]\n",
      "extrinsic_reward :  3 intrinsic_reward :  428.59515380859375\n",
      "place 138\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 138, 'x_pos_screen': 104, 'y_pos': 97}\n",
      "global_step :  66 , action :  1 action_prob :  [0.05743331089615822, 0.17481683194637299, 0.0420597642660141, 0.569418728351593, 0.07015956193208694, 0.0407501719892025, 0.04536167532205582]\n",
      "extrinsic_reward :  3 intrinsic_reward :  428.28131103515625\n",
      "place 141\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 141, 'x_pos_screen': 105, 'y_pos': 92}\n",
      "global_step :  67 , action :  3 action_prob :  [0.056843750178813934, 0.1743701994419098, 0.04155677556991577, 0.5725853443145752, 0.06956753134727478, 0.04022586718201637, 0.04485057294368744]\n",
      "extrinsic_reward :  3 intrinsic_reward :  426.5777587890625\n",
      "place 144\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 144, 'x_pos_screen': 106, 'y_pos': 87}\n",
      "global_step :  68 , action :  3 action_prob :  [0.056642383337020874, 0.17420624196529388, 0.04138093814253807, 0.5736967325210571, 0.06935533881187439, 0.04004978761076927, 0.04466848820447922]\n",
      "extrinsic_reward :  3 intrinsic_reward :  418.67742919921875\n",
      "place 147\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 147, 'x_pos_screen': 107, 'y_pos': 82}\n",
      "global_step :  69 , action :  3 action_prob :  [0.057251960039138794, 0.17466500401496887, 0.04190316051244736, 0.5704200267791748, 0.06997166574001312, 0.04058710113167763, 0.045201096683740616]\n",
      "extrinsic_reward :  3 intrinsic_reward :  423.4619445800781\n",
      "place 150\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 150, 'x_pos_screen': 108, 'y_pos': 79}\n",
      "global_step :  70 , action :  3 action_prob :  [0.05677865073084831, 0.1743122786283493, 0.04150019586086273, 0.5729497671127319, 0.06949953734874725, 0.04016643390059471, 0.044793110340833664]\n",
      "extrinsic_reward :  3 intrinsic_reward :  425.3443603515625\n",
      "place 153\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 153, 'x_pos_screen': 109, 'y_pos': 79}\n",
      "global_step :  71 , action :  2 action_prob :  [0.0561884343624115, 0.17385254800319672, 0.04099685698747635, 0.5761350989341736, 0.06890223175287247, 0.03964538872241974, 0.0442795492708683]\n",
      "extrinsic_reward :  3 intrinsic_reward :  437.049072265625\n",
      "place 156\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 156, 'x_pos_screen': 110, 'y_pos': 79}\n",
      "global_step :  72 , action :  0 action_prob :  [0.05598253756761551, 0.17369255423545837, 0.04081443324685097, 0.5772718191146851, 0.06867893785238266, 0.0394723154604435, 0.044087450951337814]\n",
      "extrinsic_reward :  3 intrinsic_reward :  432.0634765625\n",
      "place 159\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 159, 'x_pos_screen': 111, 'y_pos': 85}\n",
      "global_step :  73 , action :  1 action_prob :  [0.05660736560821533, 0.17417126893997192, 0.04135000333189964, 0.5739006996154785, 0.06931637227535248, 0.040018193423748016, 0.04463605955243111]\n",
      "extrinsic_reward :  3 intrinsic_reward :  433.0023498535156\n",
      "place 162\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 162, 'x_pos_screen': 112, 'y_pos': 91}\n",
      "global_step :  74 , action :  5 action_prob :  [0.05624271184206009, 0.17389127612113953, 0.04104054719209671, 0.5758564472198486, 0.06895215064287186, 0.03969331458210945, 0.04432351887226105]\n",
      "extrinsic_reward :  3 intrinsic_reward :  437.4241027832031\n",
      "place 165\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 165, 'x_pos_screen': 112, 'y_pos': 96}\n",
      "global_step :  75 , action :  3 action_prob :  [0.05569784715771675, 0.17345833778381348, 0.0405765026807785, 0.5788054466247559, 0.06839833408594131, 0.039214521646499634, 0.04384901002049446]\n",
      "extrinsic_reward :  3 intrinsic_reward :  448.55792236328125\n",
      "place 168\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 168, 'x_pos_screen': 112, 'y_pos': 100}\n",
      "global_step :  76 , action :  3 action_prob :  [0.05605458840727806, 0.17374810576438904, 0.04087698832154274, 0.5768808722496033, 0.06875342130661011, 0.039533667266368866, 0.04415241256356239]\n",
      "extrinsic_reward :  3 intrinsic_reward :  431.88348388671875\n",
      "place 171\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 171, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  77 , action :  3 action_prob :  [0.055782727897167206, 0.17353540658950806, 0.040647901594638824, 0.5783385038375854, 0.06848245114088058, 0.039292898029088974, 0.04392016679048538]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.77362060546875\n",
      "place 174\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 174, 'x_pos_screen': 112, 'y_pos': 106}\n",
      "global_step :  78 , action :  2 action_prob :  [0.0555226132273674, 0.17333371937274933, 0.04042136296629906, 0.5797590017318726, 0.06820710748434067, 0.0390729196369648, 0.04368327930569649]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.4026184082031\n",
      "place 177\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 177, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  79 , action :  3 action_prob :  [0.05570823699235916, 0.17348270118236542, 0.04057629406452179, 0.5787656307220459, 0.0683889165520668, 0.039239831268787384, 0.04383840411901474]\n",
      "extrinsic_reward :  2 intrinsic_reward :  454.58160400390625\n",
      "place 180\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 180, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  80 , action :  3 action_prob :  [0.05543500557541847, 0.1732795238494873, 0.0403423048555851, 0.5802320241928101, 0.06810808926820755, 0.039006028324365616, 0.04359705001115799]\n",
      "extrinsic_reward :  3 intrinsic_reward :  453.4148254394531\n",
      "place 183\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 183, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  81 , action :  3 action_prob :  [0.05501396954059601, 0.17293977737426758, 0.03998250886797905, 0.5825213193893433, 0.06767451763153076, 0.03864143788814545, 0.043226517736911774]\n",
      "extrinsic_reward :  3 intrinsic_reward :  453.40594482421875\n",
      "place 186\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 186, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  82 , action :  3 action_prob :  [0.055488377809524536, 0.1733284741640091, 0.04038642719388008, 0.5799420475959778, 0.06815847009420395, 0.03905677795410156, 0.04363948479294777]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.7176513671875\n",
      "place 189\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 189, 'x_pos_screen': 112, 'y_pos': 107}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  83 , action :  0 action_prob :  [0.054591696709394455, 0.17259454727172852, 0.03962535411119461, 0.5848092436790466, 0.06724532693624496, 0.038271721452474594, 0.04286215454339981]\n",
      "extrinsic_reward :  3 intrinsic_reward :  451.7796630859375\n",
      "place 192\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 192, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  84 , action :  3 action_prob :  [0.05494478717446327, 0.1729026585817337, 0.03991811349987984, 0.5828953385353088, 0.06759250909090042, 0.038592059165239334, 0.04315464571118355]\n",
      "extrinsic_reward :  3 intrinsic_reward :  450.0416259765625\n",
      "place 195\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 195, 'x_pos_screen': 112, 'y_pos': 101}\n",
      "global_step :  85 , action :  3 action_prob :  [0.054663728922605515, 0.17266932129859924, 0.03968221694231033, 0.584415853023529, 0.0673101618885994, 0.03834294155240059, 0.04291575029492378]\n",
      "extrinsic_reward :  3 intrinsic_reward :  453.98236083984375\n",
      "place 198\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 198, 'x_pos_screen': 112, 'y_pos': 97}\n",
      "global_step :  86 , action :  1 action_prob :  [0.05482247471809387, 0.1728006899356842, 0.03981352224946022, 0.5835663676261902, 0.06746428459882736, 0.038486797362565994, 0.04304580017924309]\n",
      "extrinsic_reward :  3 intrinsic_reward :  450.28240966796875\n",
      "place 201\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 201, 'x_pos_screen': 112, 'y_pos': 92}\n",
      "global_step :  87 , action :  6 action_prob :  [0.0548483245074749, 0.17283667623996735, 0.039825912564992905, 0.5834429860115051, 0.06747259944677353, 0.038522765040397644, 0.04305075854063034]\n",
      "extrinsic_reward :  3 intrinsic_reward :  453.3448791503906\n",
      "place 204\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 204, 'x_pos_screen': 112, 'y_pos': 87}\n",
      "global_step :  88 , action :  3 action_prob :  [0.05485687777400017, 0.17283986508846283, 0.03983795642852783, 0.583383321762085, 0.06749118864536285, 0.03852274641394615, 0.043068014085292816]\n",
      "extrinsic_reward :  3 intrinsic_reward :  445.7764587402344\n",
      "place 207\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 207, 'x_pos_screen': 112, 'y_pos': 82}\n",
      "global_step :  89 , action :  1 action_prob :  [0.05461322143673897, 0.17264284193515778, 0.039626941084861755, 0.5847206115722656, 0.0672319158911705, 0.03831763193011284, 0.042846810072660446]\n",
      "extrinsic_reward :  3 intrinsic_reward :  450.30926513671875\n",
      "place 210\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 210, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  90 , action :  3 action_prob :  [0.05499036982655525, 0.17294734716415405, 0.03995097428560257, 0.5826635956764221, 0.06762537360191345, 0.038639649748802185, 0.0431826114654541]\n",
      "extrinsic_reward :  2 intrinsic_reward :  448.1341857910156\n",
      "place 212\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 212, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  91 , action :  3 action_prob :  [0.05429310351610184, 0.17237161099910736, 0.039357226341962814, 0.5864618420600891, 0.06690806895494461, 0.03803485631942749, 0.042573291808366776]\n",
      "extrinsic_reward :  2 intrinsic_reward :  455.9737854003906\n",
      "place 214\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 214, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  92 , action :  3 action_prob :  [0.05480682849884033, 0.17278872430324554, 0.039793744683265686, 0.583676278591156, 0.06743403524160385, 0.038480207324028015, 0.04302017018198967]\n",
      "extrinsic_reward :  2 intrinsic_reward :  446.3298034667969\n",
      "place 216\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 216, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  93 , action :  4 action_prob :  [0.05508842319250107, 0.17302022874355316, 0.04002765193581581, 0.5821645855903625, 0.06771159917116165, 0.038731660693883896, 0.043255772441625595]\n",
      "extrinsic_reward :  2 intrinsic_reward :  447.6742248535156\n",
      "place 218\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 218, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  94 , action :  1 action_prob :  [0.054594386368989944, 0.1726263016462326, 0.03961154818534851, 0.5848211646080017, 0.06721502542495728, 0.03829796984791756, 0.042833562940359116]\n",
      "extrinsic_reward :  2 intrinsic_reward :  449.2979736328125\n",
      "place 220\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 220, 'x_pos_screen': 112, 'y_pos': 85}\n",
      "global_step :  95 , action :  3 action_prob :  [0.05441928282380104, 0.17247386276721954, 0.039462197571992874, 0.5857862830162048, 0.06703313440084457, 0.0381457582116127, 0.0426795668900013]\n",
      "extrinsic_reward :  3 intrinsic_reward :  450.5377197265625\n",
      "place 223\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 223, 'x_pos_screen': 112, 'y_pos': 91}\n",
      "global_step :  96 , action :  3 action_prob :  [0.055071521550416946, 0.17299394309520721, 0.040018804371356964, 0.5822516083717346, 0.06770467758178711, 0.038707662373781204, 0.043251726776361465]\n",
      "extrinsic_reward :  2 intrinsic_reward :  447.52490234375\n",
      "place 225\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 225, 'x_pos_screen': 112, 'y_pos': 96}\n",
      "global_step :  97 , action :  3 action_prob :  [0.055259037762880325, 0.1731576919555664, 0.04017677903175354, 0.58122318983078, 0.0678962767124176, 0.03887258842587471, 0.04341449216008186]\n",
      "extrinsic_reward :  2 intrinsic_reward :  447.30938720703125\n",
      "place 227\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 227, 'x_pos_screen': 112, 'y_pos': 100}\n",
      "global_step :  98 , action :  3 action_prob :  [0.05523079261183739, 0.17313405871391296, 0.040155332535505295, 0.5813674330711365, 0.06787233799695969, 0.03884638100862503, 0.04339371994137764]\n",
      "extrinsic_reward :  3 intrinsic_reward :  461.0761413574219\n",
      "place 230\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 230, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  99 , action :  3 action_prob :  [0.05533050745725632, 0.17319464683532715, 0.040240779519081116, 0.580848217010498, 0.0679750144481659, 0.03892766311764717, 0.04348309338092804]\n",
      "extrinsic_reward :  2 intrinsic_reward :  441.71405029296875\n",
      "place 233\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 233, 'x_pos_screen': 112, 'y_pos': 106}\n",
      "global_step :  100 , action :  5 action_prob :  [0.05576362460851669, 0.1735355406999588, 0.04060927405953407, 0.5785105228424072, 0.06841495633125305, 0.039306968450546265, 0.043859124183654785]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.6484375\n",
      "place 236\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 236, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  101 , action :  3 action_prob :  [0.055511247366666794, 0.17333941161632538, 0.04039902612566948, 0.5798518061637878, 0.06816911697387695, 0.03908044099807739, 0.04364902898669243]\n",
      "extrinsic_reward :  2 intrinsic_reward :  450.90496826171875\n",
      "place 238\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 238, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  102 , action :  3 action_prob :  [0.05565892904996872, 0.173436239361763, 0.04052574187517166, 0.5790737867355347, 0.06832107901573181, 0.03920340538024902, 0.043780919164419174]\n",
      "extrinsic_reward :  3 intrinsic_reward :  449.9925537109375\n",
      "place 241\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 241, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  103 , action :  5 action_prob :  [0.05601494759321213, 0.17371681332588196, 0.04082736000418663, 0.5771574974060059, 0.068678118288517, 0.03951890766620636, 0.04408631846308708]\n",
      "extrinsic_reward :  3 intrinsic_reward :  452.21722412109375\n",
      "place 244\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 244, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  104 , action :  3 action_prob :  [0.05566394701600075, 0.1734493374824524, 0.04053228721022606, 0.579026460647583, 0.06833156943321228, 0.03920703008770943, 0.04378941282629967]\n",
      "extrinsic_reward :  3 intrinsic_reward :  452.16339111328125\n",
      "place 247\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 247, 'x_pos_screen': 112, 'y_pos': 107}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  105 , action :  4 action_prob :  [0.05578188970685005, 0.17352299392223358, 0.04063182696700096, 0.5784144997596741, 0.06845078617334366, 0.03930482640862465, 0.04389321058988571]\n",
      "extrinsic_reward :  3 intrinsic_reward :  451.19366455078125\n",
      "place 250\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 250, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  106 , action :  0 action_prob :  [0.05580052733421326, 0.17355948686599731, 0.040646836161613464, 0.5782896876335144, 0.06846961379051208, 0.03932600095868111, 0.04390788450837135]\n",
      "extrinsic_reward :  3 intrinsic_reward :  448.2415771484375\n",
      "place 253\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 253, 'x_pos_screen': 112, 'y_pos': 101}\n",
      "global_step :  107 , action :  3 action_prob :  [0.05518963932991028, 0.1730562001466751, 0.04012763872742653, 0.5816138386726379, 0.06784699112176895, 0.03878888487815857, 0.04337683320045471]\n",
      "extrinsic_reward :  3 intrinsic_reward :  439.1793212890625\n",
      "place 256\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 256, 'x_pos_screen': 112, 'y_pos': 97}\n",
      "global_step :  108 , action :  3 action_prob :  [0.05578737333416939, 0.1735301911830902, 0.040634430944919586, 0.5783877372741699, 0.06845393031835556, 0.03931046649813652, 0.04389588162302971]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.31390380859375\n",
      "place 259\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 259, 'x_pos_screen': 112, 'y_pos': 92}\n",
      "global_step :  109 , action :  3 action_prob :  [0.05541510507464409, 0.1732429713010788, 0.040314607322216034, 0.5804013609886169, 0.06806997209787369, 0.03898964822292328, 0.04356631636619568]\n",
      "extrinsic_reward :  3 intrinsic_reward :  449.72711181640625\n",
      "place 262\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 262, 'x_pos_screen': 112, 'y_pos': 87}\n",
      "global_step :  110 , action :  0 action_prob :  [0.055501967668533325, 0.17330747842788696, 0.040385641157627106, 0.5799484252929688, 0.06815216690301895, 0.03906736895442009, 0.043636977672576904]\n",
      "extrinsic_reward :  3 intrinsic_reward :  439.65032958984375\n",
      "place 265\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 265, 'x_pos_screen': 112, 'y_pos': 82}\n",
      "global_step :  111 , action :  5 action_prob :  [0.05588192492723465, 0.1736019104719162, 0.04070763289928436, 0.5779085755348206, 0.06853444874286652, 0.039401598274707794, 0.04396390542387962]\n",
      "extrinsic_reward :  3 intrinsic_reward :  440.98052978515625\n",
      "place 268\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 268, 'x_pos_screen': 112, 'y_pos': 77}\n",
      "global_step :  112 , action :  3 action_prob :  [0.05549538508057594, 0.17330175638198853, 0.04038388282060623, 0.5799704790115356, 0.06815293431282043, 0.03905842453241348, 0.04363719001412392]\n",
      "extrinsic_reward :  3 intrinsic_reward :  459.0018615722656\n",
      "place 271\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 271, 'x_pos_screen': 112, 'y_pos': 72}\n",
      "global_step :  113 , action :  3 action_prob :  [0.05533021315932274, 0.17317230999469757, 0.04023921489715576, 0.5808789134025574, 0.06797412037849426, 0.03892193362116814, 0.04348331689834595]\n",
      "extrinsic_reward :  3 intrinsic_reward :  448.609619140625\n",
      "place 274\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 274, 'x_pos_screen': 112, 'y_pos': 67}\n",
      "global_step :  114 , action :  0 action_prob :  [0.05544276535511017, 0.1732674390077591, 0.04033515974879265, 0.5802620053291321, 0.06809022277593613, 0.03902055695652962, 0.04358185455203056]\n",
      "extrinsic_reward :  3 intrinsic_reward :  448.8288269042969\n",
      "place 277\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 277, 'x_pos_screen': 112, 'y_pos': 62}\n",
      "global_step :  115 , action :  5 action_prob :  [0.054800909012556076, 0.17275287210941315, 0.03978600353002548, 0.5837509632110596, 0.06742637604475021, 0.038467682898044586, 0.043015096336603165]\n",
      "extrinsic_reward :  3 intrinsic_reward :  442.57940673828125\n",
      "place 280\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 280, 'x_pos_screen': 112, 'y_pos': 57}\n",
      "global_step :  116 , action :  4 action_prob :  [0.0550277940928936, 0.17294903099536896, 0.03997392579913139, 0.5825268626213074, 0.06764602661132812, 0.03867579996585846, 0.04320044815540314]\n",
      "extrinsic_reward :  3 intrinsic_reward :  443.7865295410156\n",
      "place 283\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 283, 'x_pos_screen': 112, 'y_pos': 52}\n",
      "global_step :  117 , action :  3 action_prob :  [0.05480536073446274, 0.17275092005729675, 0.039787907153367996, 0.5837439894676208, 0.06742367148399353, 0.0384758897125721, 0.043012287467718124]\n",
      "extrinsic_reward :  3 intrinsic_reward :  448.3819885253906\n",
      "place 286\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 286, 'x_pos_screen': 112, 'y_pos': 47}\n",
      "global_step :  118 , action :  4 action_prob :  [0.0548725388944149, 0.17282713949680328, 0.03983708471059799, 0.5833888053894043, 0.06747143715620041, 0.038553088903427124, 0.04304985702037811]\n",
      "extrinsic_reward :  3 intrinsic_reward :  438.6397705078125\n",
      "place 289\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 289, 'x_pos_screen': 112, 'y_pos': 42}\n",
      "global_step :  119 , action :  3 action_prob :  [0.05504123494029045, 0.17295031249523163, 0.03998102992773056, 0.582489013671875, 0.06764402240514755, 0.03869660943746567, 0.04319774731993675]\n",
      "extrinsic_reward :  2 intrinsic_reward :  426.89727783203125\n",
      "place 292\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 292, 'x_pos_screen': 112, 'y_pos': 37}\n",
      "global_step :  120 , action :  1 action_prob :  [0.05464167147874832, 0.17261557281017303, 0.039647217839956284, 0.5846465229988098, 0.06724730134010315, 0.03833978623151779, 0.04286182299256325]\n",
      "extrinsic_reward :  3 intrinsic_reward :  440.905029296875\n",
      "place 295\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 295, 'x_pos_screen': 112, 'y_pos': 32}\n",
      "global_step :  121 , action :  0 action_prob :  [0.05461477115750313, 0.17259833216667175, 0.039618782699108124, 0.5848158597946167, 0.06720118224620819, 0.038330305367708206, 0.04282081872224808]\n",
      "extrinsic_reward :  3 intrinsic_reward :  429.9741516113281\n",
      "place 298\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 298, 'x_pos_screen': 112, 'y_pos': 27}\n",
      "global_step :  122 , action :  2 action_prob :  [0.05468883737921715, 0.17265795171260834, 0.03968163952231407, 0.5844128131866455, 0.06727916747331619, 0.03839270398020744, 0.0428868792951107]\n",
      "extrinsic_reward :  3 intrinsic_reward :  429.54571533203125\n",
      "place 301\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 301, 'x_pos_screen': 112, 'y_pos': 22}\n",
      "global_step :  123 , action :  3 action_prob :  [0.05432017147541046, 0.17236264050006866, 0.039364468306303024, 0.5864262580871582, 0.06688914448022842, 0.03808233514428139, 0.042555004358291626]\n",
      "extrinsic_reward :  3 intrinsic_reward :  415.8267822265625\n",
      "place 304\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 304, 'x_pos_screen': 112, 'y_pos': 17}\n",
      "global_step :  124 , action :  3 action_prob :  [0.05452645197510719, 0.1725437343120575, 0.039534468203783035, 0.5853142738342285, 0.06708738207817078, 0.0382734090089798, 0.042720336467027664]\n",
      "extrinsic_reward :  3 intrinsic_reward :  429.8251037597656\n",
      "place 307\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 307, 'x_pos_screen': 112, 'y_pos': 12}\n",
      "global_step :  125 , action :  3 action_prob :  [0.05410691350698471, 0.17216336727142334, 0.039186954498291016, 0.5876030325889587, 0.06667403876781464, 0.037892453372478485, 0.04237324371933937]\n",
      "extrinsic_reward :  3 intrinsic_reward :  426.7162170410156\n",
      "place 310\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 310, 'x_pos_screen': 112, 'y_pos': 7}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  126 , action :  3 action_prob :  [0.05485275015234947, 0.17279431223869324, 0.03980395942926407, 0.5835942625999451, 0.06740225106477737, 0.03856537118554115, 0.04298708960413933]\n",
      "extrinsic_reward :  3 intrinsic_reward :  399.610107421875\n",
      "place 313\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 313, 'x_pos_screen': 112, 'y_pos': 2}\n",
      "epoch :  2 , global_step :  127\n",
      "global_step :  1 , action :  4 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.56402587890625\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  2 , action :  4 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.56402587890625\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 84}\n",
      "global_step :  3 , action :  3 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  413.3307800292969\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 89}\n",
      "global_step :  4 , action :  4 action_prob :  [0.05978989228606224, 0.1765068620443344, 0.04407934099435806, 0.5568422079086304, 0.07250624150037766, 0.04286649823188782, 0.04740886762738228]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.8639221191406\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 93}\n",
      "global_step :  5 , action :  1 action_prob :  [0.05976428836584091, 0.17649048566818237, 0.044056519865989685, 0.5569798350334167, 0.0724792331457138, 0.04284464567899704, 0.04738491773605347]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.188232421875\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 97}\n",
      "global_step :  6 , action :  3 action_prob :  [0.05968368053436279, 0.17643778026103973, 0.04398546367883682, 0.5574111938476562, 0.07239601016044617, 0.04277447983622551, 0.047311339527368546]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.49737548828125\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 100}\n",
      "global_step :  7 , action :  3 action_prob :  [0.059704381972551346, 0.1764526069164276, 0.0440034382045269, 0.5572999119758606, 0.07241684198379517, 0.042793162167072296, 0.0473296232521534]\n",
      "extrinsic_reward :  0 intrinsic_reward :  414.0\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 103}\n",
      "global_step :  8 , action :  3 action_prob :  [0.05966486409306526, 0.17642614245414734, 0.04396839812397957, 0.5575129389762878, 0.07237570732831955, 0.04275871068239212, 0.04729333147406578]\n",
      "extrinsic_reward :  1 intrinsic_reward :  413.5565185546875\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 104}\n",
      "global_step :  9 , action :  3 action_prob :  [0.059683870524168015, 0.17643922567367554, 0.043984923511743546, 0.557411253452301, 0.07239478081464767, 0.04277579486370087, 0.04731012508273125]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.0807189941406\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 106}\n",
      "global_step :  10 , action :  6 action_prob :  [0.05972190201282501, 0.17646507918834686, 0.04401828348636627, 0.5572071671485901, 0.07243375480175018, 0.04280931130051613, 0.04734450951218605]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.66156005859375\n",
      "place 43\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 43, 'x_pos_screen': 43, 'y_pos': 106}\n",
      "global_step :  11 , action :  1 action_prob :  [0.059727638959884644, 0.17646794021129608, 0.04402357339859009, 0.5571768283843994, 0.07243993133306503, 0.04281395301222801, 0.04735007882118225]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.006591796875\n",
      "place 43\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 43, 'x_pos_screen': 43, 'y_pos': 106}\n",
      "global_step :  12 , action :  3 action_prob :  [0.05978003144264221, 0.17650336027145386, 0.04406954348087311, 0.556895911693573, 0.07249358296394348, 0.04286021739244461, 0.047397371381521225]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.006591796875\n",
      "place 44\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 44, 'x_pos_screen': 44, 'y_pos': 105}\n",
      "global_step :  13 , action :  6 action_prob :  [0.05978003144264221, 0.17650336027145386, 0.04406954348087311, 0.556895911693573, 0.07249358296394348, 0.04286021739244461, 0.047397371381521225]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.58660888671875\n",
      "place 45\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 45, 'x_pos_screen': 45, 'y_pos': 104}\n",
      "global_step :  14 , action :  3 action_prob :  [0.059778232127428055, 0.1765020787715912, 0.0440681017935276, 0.5569050312042236, 0.07249199599027634, 0.04285844787955284, 0.04739600792527199]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.2864990234375\n",
      "place 45\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 45, 'x_pos_screen': 45, 'y_pos': 101}\n",
      "global_step :  15 , action :  3 action_prob :  [0.05980805307626724, 0.1765228807926178, 0.044094014912843704, 0.5567452907562256, 0.0725221112370491, 0.042885202914476395, 0.04742247611284256]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.42626953125\n",
      "place 46\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 46, 'x_pos_screen': 46, 'y_pos': 99}\n",
      "global_step :  16 , action :  2 action_prob :  [0.05974053591489792, 0.17647723853588104, 0.044034600257873535, 0.5571079850196838, 0.07245275378227234, 0.04282567277550697, 0.04736127704381943]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.6059265136719\n",
      "place 47\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 47, 'x_pos_screen': 47, 'y_pos': 95}\n",
      "global_step :  17 , action :  3 action_prob :  [0.059752631932497025, 0.1764848679304123, 0.04404561594128609, 0.5570423603057861, 0.07246582210063934, 0.0428357794880867, 0.04737294092774391]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.5221252441406\n",
      "place 48\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 48, 'x_pos_screen': 48, 'y_pos': 91}\n",
      "global_step :  18 , action :  3 action_prob :  [0.059734366834163666, 0.17647293210029602, 0.04402963072061539, 0.5571395754814148, 0.07244724035263062, 0.0428197905421257, 0.04735647514462471]\n",
      "extrinsic_reward :  1 intrinsic_reward :  408.5665588378906\n",
      "place 49\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 49, 'x_pos_screen': 49, 'y_pos': 86}\n",
      "global_step :  19 , action :  5 action_prob :  [0.05975338816642761, 0.17648522555828094, 0.04404665529727936, 0.5570370554924011, 0.07246734946966171, 0.042836084961891174, 0.047374263405799866]\n",
      "extrinsic_reward :  -1 intrinsic_reward :  410.36376953125\n",
      "place 49\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 49, 'x_pos_screen': 49, 'y_pos': 81}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  20 , action :  1 action_prob :  [0.059759192168712616, 0.17648889124393463, 0.04405176267027855, 0.5570061802864075, 0.07247326523065567, 0.042841147631406784, 0.04737953469157219]\n",
      "extrinsic_reward :  1 intrinsic_reward :  409.74365234375\n",
      "place 50\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 50, 'x_pos_screen': 50, 'y_pos': 79}\n",
      "global_step :  21 , action :  3 action_prob :  [0.059839654713869095, 0.17654338479042053, 0.0441221185028553, 0.556575357913971, 0.07255519181489944, 0.04291249439120293, 0.04745174199342728]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.8468322753906\n",
      "place 51\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 51, 'x_pos_screen': 51, 'y_pos': 79}\n",
      "global_step :  22 , action :  3 action_prob :  [0.05985831096768379, 0.1765560656785965, 0.04413874074816704, 0.5564744472503662, 0.07257462292909622, 0.042928896844387054, 0.0474688820540905]\n",
      "extrinsic_reward :  1 intrinsic_reward :  409.69305419921875\n",
      "place 52\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 52, 'x_pos_screen': 52, 'y_pos': 79}\n",
      "global_step :  23 , action :  4 action_prob :  [0.05986681208014488, 0.17656221985816956, 0.04414632171392441, 0.5564278364181519, 0.07258360087871552, 0.04293639212846756, 0.04747675359249115]\n",
      "extrinsic_reward :  2 intrinsic_reward :  410.3902893066406\n",
      "place 54\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 54, 'x_pos_screen': 54, 'y_pos': 79}\n",
      "global_step :  24 , action :  5 action_prob :  [0.05985331907868385, 0.17655277252197266, 0.04413452371954918, 0.5565003752708435, 0.072569839656353, 0.042924366891384125, 0.047464653849601746]\n",
      "extrinsic_reward :  1 intrinsic_reward :  409.3848571777344\n",
      "place 55\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 55, 'x_pos_screen': 55, 'y_pos': 84}\n",
      "global_step :  25 , action :  3 action_prob :  [0.059816379100084305, 0.17652790248394012, 0.044102225452661514, 0.5566982626914978, 0.07253216952085495, 0.042891744524240494, 0.04743138700723648]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.814453125\n",
      "place 56\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 56, 'x_pos_screen': 56, 'y_pos': 89}\n",
      "global_step :  26 , action :  3 action_prob :  [0.05980177968740463, 0.17651793360710144, 0.04408872872591019, 0.556779146194458, 0.07251592725515366, 0.042879387736320496, 0.04741709679365158]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.590576171875\n",
      "place 57\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 57, 'x_pos_screen': 57, 'y_pos': 93}\n",
      "global_step :  27 , action :  4 action_prob :  [0.05987927317619324, 0.17657071352005005, 0.044156648218631744, 0.5563633441925049, 0.07259508967399597, 0.04294800013303757, 0.047486916184425354]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.03875732421875\n",
      "place 59\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 59, 'x_pos_screen': 59, 'y_pos': 97}\n",
      "global_step :  28 , action :  1 action_prob :  [0.05979998782277107, 0.17651709914207458, 0.04408715292811394, 0.5567883253097534, 0.07251402735710144, 0.042877957224845886, 0.047415412962436676]\n",
      "extrinsic_reward :  1 intrinsic_reward :  413.5328674316406\n",
      "place 60\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 60, 'x_pos_screen': 60, 'y_pos': 101}\n",
      "global_step :  29 , action :  3 action_prob :  [0.05983422324061394, 0.1765407919883728, 0.04411683231592178, 0.5566053986549377, 0.07254843413829803, 0.04290865361690521, 0.047445718199014664]\n",
      "extrinsic_reward :  2 intrinsic_reward :  410.5879821777344\n",
      "place 62\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 62, 'x_pos_screen': 62, 'y_pos': 103}\n",
      "global_step :  30 , action :  3 action_prob :  [0.05978615954518318, 0.1765078455209732, 0.04407446086406708, 0.5568645596504211, 0.07249877601861954, 0.042866308242082596, 0.04740196838974953]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.5341491699219\n",
      "place 63\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 63, 'x_pos_screen': 63, 'y_pos': 106}\n",
      "global_step :  31 , action :  3 action_prob :  [0.0598260834813118, 0.17653532326221466, 0.04410934820771217, 0.5566501617431641, 0.07253944873809814, 0.04290180280804634, 0.04743778333067894]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.9395446777344\n",
      "place 65\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 65, 'x_pos_screen': 65, 'y_pos': 107}\n",
      "global_step :  32 , action :  2 action_prob :  [0.05981704220175743, 0.17652738094329834, 0.04410278797149658, 0.556695818901062, 0.07253281772136688, 0.042891860008239746, 0.04743216186761856]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.2336730957031\n",
      "place 67\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 67, 'x_pos_screen': 67, 'y_pos': 108}\n",
      "global_step :  33 , action :  3 action_prob :  [0.059783805161714554, 0.17650443315505981, 0.04407389089465141, 0.5568736791610718, 0.07249929010868073, 0.04286215454339981, 0.04740263894200325]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.25921630859375\n",
      "place 68\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 68, 'x_pos_screen': 68, 'y_pos': 109}\n",
      "global_step :  34 , action :  3 action_prob :  [0.05976901203393936, 0.1764940768480301, 0.04406077042222023, 0.5569541454315186, 0.07248374074697495, 0.04284929111599922, 0.047388944774866104]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.2796325683594\n",
      "place 70\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 70, 'x_pos_screen': 70, 'y_pos': 109}\n",
      "global_step :  35 , action :  3 action_prob :  [0.059830162674188614, 0.1765356808900833, 0.0441146083176136, 0.5566251873970032, 0.07254672795534134, 0.0429031141102314, 0.04744448512792587]\n",
      "extrinsic_reward :  2 intrinsic_reward :  410.9110412597656\n",
      "place 72\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 72, 'x_pos_screen': 72, 'y_pos': 108}\n",
      "global_step :  36 , action :  1 action_prob :  [0.059846412390470505, 0.176548570394516, 0.04412756487727165, 0.5565405488014221, 0.07256074249744415, 0.04291955381631851, 0.04745659604668617]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.03070068359375\n",
      "place 74\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 74, 'x_pos_screen': 74, 'y_pos': 107}\n",
      "global_step :  37 , action :  3 action_prob :  [0.05989154055714607, 0.17657971382141113, 0.04416641220450401, 0.5563006401062012, 0.07260521501302719, 0.04296061396598816, 0.047495752573013306]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.3003234863281\n",
      "place 75\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 75, 'x_pos_screen': 75, 'y_pos': 105}\n",
      "global_step :  38 , action :  3 action_prob :  [0.059903837740421295, 0.17658744752407074, 0.04417775571346283, 0.5562334060668945, 0.07261903584003448, 0.04297057166695595, 0.04750801995396614]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.0520935058594\n",
      "place 77\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 77, 'x_pos_screen': 77, 'y_pos': 103}\n",
      "global_step :  39 , action :  1 action_prob :  [0.05988951772451401, 0.176577627658844, 0.04416521638631821, 0.55631023645401, 0.07260434329509735, 0.04295793175697327, 0.04749506711959839]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.469482421875\n",
      "place 79\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 79, 'x_pos_screen': 79, 'y_pos': 100}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  40 , action :  1 action_prob :  [0.059975750744342804, 0.17663626372814178, 0.04424053430557251, 0.5558488368988037, 0.07269176840782166, 0.043034639209508896, 0.047572217881679535]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.823974609375\n",
      "place 81\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 81, 'x_pos_screen': 81, 'y_pos': 97}\n",
      "global_step :  41 , action :  3 action_prob :  [0.06000417470932007, 0.17665478587150574, 0.04426584020256996, 0.5556958317756653, 0.07272151112556458, 0.043059173971414566, 0.047598566859960556]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.18255615234375\n",
      "place 82\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 82, 'x_pos_screen': 81, 'y_pos': 93}\n",
      "global_step :  42 , action :  4 action_prob :  [0.05996882542967796, 0.1766292303800583, 0.04423557221889496, 0.555884838104248, 0.0726868137717247, 0.04302658513188362, 0.04756812006235123]\n",
      "extrinsic_reward :  2 intrinsic_reward :  398.2533264160156\n",
      "place 84\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 84, 'x_pos_screen': 82, 'y_pos': 88}\n",
      "global_step :  43 , action :  5 action_prob :  [0.05873802304267883, 0.1757761538028717, 0.04317012056708336, 0.5624585151672363, 0.07144871354103088, 0.04192719981074333, 0.046481288969516754]\n",
      "extrinsic_reward :  2 intrinsic_reward :  394.2473449707031\n",
      "place 86\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 86, 'x_pos_screen': 83, 'y_pos': 83}\n",
      "global_step :  44 , action :  4 action_prob :  [0.05974428728222847, 0.17646841704845428, 0.04405519738793373, 0.5570360422134399, 0.0724925696849823, 0.04280465841293335, 0.04739881679415703]\n",
      "extrinsic_reward :  2 intrinsic_reward :  421.78912353515625\n",
      "place 88\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 88, 'x_pos_screen': 84, 'y_pos': 79}\n",
      "global_step :  45 , action :  4 action_prob :  [0.05998304486274719, 0.17663221061229706, 0.04425844922661781, 0.5557774305343628, 0.07272268086671829, 0.04302540048956871, 0.04760076478123665]\n",
      "extrinsic_reward :  2 intrinsic_reward :  408.51507568359375\n",
      "place 90\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 90, 'x_pos_screen': 85, 'y_pos': 79}\n",
      "global_step :  46 , action :  3 action_prob :  [0.05896493047475815, 0.1759270876646042, 0.04337581992149353, 0.5612180829048157, 0.07169788330793381, 0.04211607947945595, 0.04670008271932602]\n",
      "extrinsic_reward :  2 intrinsic_reward :  426.6092834472656\n",
      "place 92\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 92, 'x_pos_screen': 86, 'y_pos': 79}\n",
      "global_step :  47 , action :  3 action_prob :  [0.060238614678382874, 0.17680723965168, 0.04447999969124794, 0.5544155240058899, 0.07297707349061966, 0.04325580969452858, 0.04782572388648987]\n",
      "extrinsic_reward :  2 intrinsic_reward :  406.1426696777344\n",
      "place 94\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 94, 'x_pos_screen': 87, 'y_pos': 79}\n",
      "global_step :  48 , action :  3 action_prob :  [0.059709902852773666, 0.17644308507442474, 0.04401972517371178, 0.5572465062141418, 0.07244305312633514, 0.04278292879462242, 0.047354795038700104]\n",
      "extrinsic_reward :  2 intrinsic_reward :  417.008056640625\n",
      "place 96\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 96, 'x_pos_screen': 88, 'y_pos': 79}\n",
      "global_step :  49 , action :  4 action_prob :  [0.06056774780154228, 0.17701683938503265, 0.044764794409275055, 0.5526832938194275, 0.07330147922039032, 0.04355267807841301, 0.04811320826411247]\n",
      "extrinsic_reward :  2 intrinsic_reward :  418.9347229003906\n",
      "place 98\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 98, 'x_pos_screen': 89, 'y_pos': 79}\n",
      "global_step :  50 , action :  3 action_prob :  [0.05952395498752594, 0.1763170063495636, 0.04385918378829956, 0.5582327246665955, 0.07225895673036575, 0.04261593893170357, 0.04719228297472]\n",
      "extrinsic_reward :  3 intrinsic_reward :  399.8058776855469\n",
      "place 101\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 101, 'x_pos_screen': 90, 'y_pos': 85}\n",
      "global_step :  51 , action :  3 action_prob :  [0.05843368172645569, 0.17555008828639984, 0.04291394725441933, 0.5640750527381897, 0.07115454971790314, 0.0416470468044281, 0.04622555896639824]\n",
      "extrinsic_reward :  2 intrinsic_reward :  420.8527526855469\n",
      "place 103\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 103, 'x_pos_screen': 91, 'y_pos': 91}\n",
      "global_step :  52 , action :  3 action_prob :  [0.059318941086530685, 0.17617268860340118, 0.04368796572089195, 0.5593066215515137, 0.07206583023071289, 0.04242470860481262, 0.047023240476846695]\n",
      "extrinsic_reward :  3 intrinsic_reward :  417.42340087890625\n",
      "place 106\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 106, 'x_pos_screen': 92, 'y_pos': 96}\n",
      "global_step :  53 , action :  3 action_prob :  [0.05842309817671776, 0.17554649710655212, 0.04290904104709625, 0.5641087293624878, 0.07115442305803299, 0.04163329675793648, 0.04622482880949974]\n",
      "extrinsic_reward :  2 intrinsic_reward :  416.1604309082031\n",
      "place 108\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 108, 'x_pos_screen': 93, 'y_pos': 100}\n",
      "global_step :  54 , action :  5 action_prob :  [0.05915938317775726, 0.1760619431734085, 0.04354608803987503, 0.5601751208305359, 0.07189564406871796, 0.04228797182440758, 0.04687395319342613]\n",
      "extrinsic_reward :  3 intrinsic_reward :  423.4576721191406\n",
      "place 111\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 111, 'x_pos_screen': 94, 'y_pos': 104}\n",
      "global_step :  55 , action :  3 action_prob :  [0.05998695641756058, 0.17662473022937775, 0.044265471398830414, 0.555759072303772, 0.07273026555776596, 0.043024562299251556, 0.047608934342861176]\n",
      "extrinsic_reward :  3 intrinsic_reward :  416.02508544921875\n",
      "place 114\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 114, 'x_pos_screen': 95, 'y_pos': 106}\n",
      "global_step :  56 , action :  3 action_prob :  [0.057935282588005066, 0.17519511282444, 0.042485371232032776, 0.566737711429596, 0.07065250724554062, 0.041206203401088715, 0.04578786715865135]\n",
      "extrinsic_reward :  3 intrinsic_reward :  432.95556640625\n",
      "place 117\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 117, 'x_pos_screen': 96, 'y_pos': 108}\n",
      "global_step :  57 , action :  5 action_prob :  [0.05853642150759697, 0.17562834918498993, 0.0430113710463047, 0.5634836554527283, 0.07127797603607178, 0.041729241609573364, 0.04633302241563797]\n",
      "extrinsic_reward :  3 intrinsic_reward :  432.823486328125\n",
      "place 120\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 120, 'x_pos_screen': 97, 'y_pos': 109}\n",
      "global_step :  58 , action :  3 action_prob :  [0.058816149830818176, 0.17582900822162628, 0.04325290769338608, 0.5619854927062988, 0.07155866920948029, 0.041978854686021805, 0.04657895863056183]\n",
      "extrinsic_reward :  3 intrinsic_reward :  425.15753173828125\n",
      "place 123\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 123, 'x_pos_screen': 98, 'y_pos': 109}\n",
      "global_step :  59 , action :  3 action_prob :  [0.059072136878967285, 0.1760081946849823, 0.043471597135066986, 0.5606282949447632, 0.07181030511856079, 0.04221058264374733, 0.04679897800087929]\n",
      "extrinsic_reward :  2 intrinsic_reward :  409.16552734375\n",
      "place 126\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 126, 'x_pos_screen': 99, 'y_pos': 108}\n",
      "global_step :  60 , action :  1 action_prob :  [0.05735732242465019, 0.17476774752140045, 0.041989024728536606, 0.569841206073761, 0.0700690895318985, 0.040693361312150955, 0.04528224840760231]\n",
      "extrinsic_reward :  3 intrinsic_reward :  433.88201904296875\n",
      "place 129\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 129, 'x_pos_screen': 100, 'y_pos': 107}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  61 , action :  3 action_prob :  [0.05781036987900734, 0.17510375380516052, 0.04238405451178551, 0.5673812627792358, 0.07054214179515839, 0.04108612984418869, 0.045692313462495804]\n",
      "extrinsic_reward :  3 intrinsic_reward :  434.6617431640625\n",
      "place 132\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 132, 'x_pos_screen': 101, 'y_pos': 104}\n",
      "global_step :  62 , action :  3 action_prob :  [0.05808495730161667, 0.17529669404029846, 0.04262257739901543, 0.565910279750824, 0.07082267105579376, 0.04132528230547905, 0.04593762382864952]\n",
      "extrinsic_reward :  3 intrinsic_reward :  431.8848876953125\n",
      "place 135\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 135, 'x_pos_screen': 102, 'y_pos': 101}\n",
      "global_step :  63 , action :  5 action_prob :  [0.05842521786689758, 0.17553554475307465, 0.04291604831814766, 0.5640950202941895, 0.07116512954235077, 0.041626207530498505, 0.04623677581548691]\n",
      "extrinsic_reward :  3 intrinsic_reward :  424.8511047363281\n",
      "place 138\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 138, 'x_pos_screen': 103, 'y_pos': 97}\n",
      "global_step :  64 , action :  3 action_prob :  [0.05688057467341423, 0.17440497875213623, 0.0415789932012558, 0.572415828704834, 0.06958413124084473, 0.0402718111872673, 0.044863685965538025]\n",
      "extrinsic_reward :  3 intrinsic_reward :  434.0258483886719\n",
      "place 141\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 141, 'x_pos_screen': 104, 'y_pos': 92}\n",
      "global_step :  65 , action :  4 action_prob :  [0.05711546912789345, 0.1745772808790207, 0.041786711663007736, 0.5711302161216736, 0.0698382779955864, 0.04046781733632088, 0.04508422315120697]\n",
      "extrinsic_reward :  3 intrinsic_reward :  441.4322509765625\n",
      "place 144\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 144, 'x_pos_screen': 105, 'y_pos': 87}\n",
      "global_step :  66 , action :  3 action_prob :  [0.057339783757925034, 0.17473559081554413, 0.04198109731078148, 0.5699307918548584, 0.07006710767745972, 0.04066188633441925, 0.04528376832604408]\n",
      "extrinsic_reward :  3 intrinsic_reward :  434.6139831542969\n",
      "place 147\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 147, 'x_pos_screen': 106, 'y_pos': 82}\n",
      "global_step :  67 , action :  3 action_prob :  [0.0576762817800045, 0.17498061060905457, 0.042270950973033905, 0.5681257843971252, 0.07040850073099136, 0.04095769301056862, 0.04558015987277031]\n",
      "extrinsic_reward :  3 intrinsic_reward :  414.5443115234375\n",
      "place 150\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 150, 'x_pos_screen': 107, 'y_pos': 79}\n",
      "global_step :  68 , action :  3 action_prob :  [0.0563809871673584, 0.17400678992271423, 0.041152261197566986, 0.5751252770423889, 0.06907862424850464, 0.039825331419706345, 0.0444306880235672]\n",
      "extrinsic_reward :  3 intrinsic_reward :  434.30743408203125\n",
      "place 153\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 153, 'x_pos_screen': 108, 'y_pos': 79}\n",
      "global_step :  69 , action :  3 action_prob :  [0.05652258172631264, 0.17411066591739655, 0.0412805899977684, 0.5743387937545776, 0.0692385882139206, 0.03994009643793106, 0.04456872120499611]\n",
      "extrinsic_reward :  3 intrinsic_reward :  438.7597961425781\n",
      "place 156\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 156, 'x_pos_screen': 109, 'y_pos': 79}\n",
      "global_step :  70 , action :  0 action_prob :  [0.05669653043150902, 0.17423921823501587, 0.04143091291189194, 0.5734025239944458, 0.0694168359041214, 0.040090691298246384, 0.04472317919135094]\n",
      "extrinsic_reward :  3 intrinsic_reward :  435.6051025390625\n",
      "place 159\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 159, 'x_pos_screen': 110, 'y_pos': 79}\n",
      "global_step :  71 , action :  3 action_prob :  [0.056986235082149506, 0.1744551956653595, 0.04167943820357323, 0.5718461871147156, 0.06971105933189392, 0.040344227105379105, 0.04497768357396126]\n",
      "extrinsic_reward :  3 intrinsic_reward :  433.337646484375\n",
      "place 162\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 162, 'x_pos_screen': 111, 'y_pos': 79}\n",
      "global_step :  72 , action :  3 action_prob :  [0.05598738417029381, 0.1736963391304016, 0.040817394852638245, 0.5772517323493958, 0.06867999583482742, 0.039478130638599396, 0.04408895596861839]\n",
      "extrinsic_reward :  3 intrinsic_reward :  446.1143798828125\n",
      "place 165\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 165, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  73 , action :  3 action_prob :  [0.055985793471336365, 0.17368531227111816, 0.04082160443067551, 0.5772501826286316, 0.06869056075811386, 0.03946758806705475, 0.04409896209836006]\n",
      "extrinsic_reward :  3 intrinsic_reward :  446.67242431640625\n",
      "place 168\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 168, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  74 , action :  5 action_prob :  [0.05601218342781067, 0.17371171712875366, 0.040841273963451385, 0.5771119594573975, 0.06871138513088226, 0.03949495777487755, 0.04411657899618149]\n",
      "extrinsic_reward :  3 intrinsic_reward :  433.6063232421875\n",
      "place 171\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 171, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  75 , action :  3 action_prob :  [0.05570153892040253, 0.17346718907356262, 0.04057816416025162, 0.5787844657897949, 0.06839890033006668, 0.03922070935368538, 0.04384908080101013]\n",
      "extrinsic_reward :  3 intrinsic_reward :  449.90899658203125\n",
      "place 174\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 174, 'x_pos_screen': 112, 'y_pos': 85}\n",
      "global_step :  76 , action :  4 action_prob :  [0.05545971915125847, 0.1732744723558426, 0.040369369089603424, 0.5801060795783997, 0.06814494729042053, 0.03901413455605507, 0.04363131895661354]\n",
      "extrinsic_reward :  3 intrinsic_reward :  444.5492248535156\n",
      "place 177\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 177, 'x_pos_screen': 112, 'y_pos': 91}\n",
      "global_step :  77 , action :  3 action_prob :  [0.05564958229660988, 0.17343012988567352, 0.04052654653787613, 0.579090416431427, 0.068328857421875, 0.03918648511171341, 0.04378794878721237]\n",
      "extrinsic_reward :  3 intrinsic_reward :  452.12615966796875\n",
      "place 180\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 180, 'x_pos_screen': 112, 'y_pos': 96}\n",
      "global_step :  78 , action :  3 action_prob :  [0.0553373321890831, 0.1731967031955719, 0.040257785469293594, 0.5807722210884094, 0.06800570338964462, 0.03891973942518234, 0.043510500341653824]\n",
      "extrinsic_reward :  3 intrinsic_reward :  448.826416015625\n",
      "place 183\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 183, 'x_pos_screen': 112, 'y_pos': 100}\n",
      "global_step :  79 , action :  3 action_prob :  [0.054925333708524704, 0.17286817729473114, 0.03990539163351059, 0.5830082893371582, 0.06758090108633041, 0.0385650098323822, 0.043146997690200806]\n",
      "extrinsic_reward :  2 intrinsic_reward :  446.98236083984375\n",
      "place 186\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 186, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  80 , action :  3 action_prob :  [0.05541804060339928, 0.17327360808849335, 0.04032518342137337, 0.5803258419036865, 0.0680847093462944, 0.038996100425720215, 0.04357646778225899]\n",
      "extrinsic_reward :  3 intrinsic_reward :  445.0888366699219\n",
      "place 189\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 189, 'x_pos_screen': 112, 'y_pos': 106}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  81 , action :  3 action_prob :  [0.05458497628569603, 0.17258691787719727, 0.03962021321058273, 0.5848464369773865, 0.06723939627408981, 0.038264501839876175, 0.042857520282268524]\n",
      "extrinsic_reward :  3 intrinsic_reward :  453.21240234375\n",
      "place 192\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 192, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  82 , action :  1 action_prob :  [0.054960478097200394, 0.17291127145290375, 0.03993238881230354, 0.5828123092651367, 0.06760989129543304, 0.038603752851486206, 0.04317000135779381]\n",
      "extrinsic_reward :  3 intrinsic_reward :  453.54638671875\n",
      "place 195\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 195, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  83 , action :  1 action_prob :  [0.054668158292770386, 0.1726752072572708, 0.03968551754951477, 0.5843904614448547, 0.06731396168470383, 0.03834807500243187, 0.04291857033967972]\n",
      "extrinsic_reward :  3 intrinsic_reward :  457.42327880859375\n",
      "place 198\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 198, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  84 , action :  3 action_prob :  [0.05484825000166893, 0.17282456159591675, 0.03983506187796593, 0.5834240913391113, 0.06748999655246735, 0.03851082921028137, 0.04306716471910477]\n",
      "extrinsic_reward :  3 intrinsic_reward :  451.4051818847656\n",
      "place 201\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 201, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  85 , action :  3 action_prob :  [0.05497341603040695, 0.17293529212474823, 0.03993497043848038, 0.5827584266662598, 0.06760486215353012, 0.03862956166267395, 0.04316350072622299]\n",
      "extrinsic_reward :  3 intrinsic_reward :  456.4073486328125\n",
      "place 204\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 204, 'x_pos_screen': 112, 'y_pos': 107}\n",
      "global_step :  86 , action :  1 action_prob :  [0.05494048446416855, 0.17291364073753357, 0.03990998491644859, 0.5829183459281921, 0.06757888942956924, 0.03859706223011017, 0.043141625821590424]\n",
      "extrinsic_reward :  3 intrinsic_reward :  449.21807861328125\n",
      "place 207\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 207, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  87 , action :  3 action_prob :  [0.054667990654706955, 0.17269505560398102, 0.039673566818237305, 0.5844126343727112, 0.0672890916466713, 0.038367416709661484, 0.04289422556757927]\n",
      "extrinsic_reward :  3 intrinsic_reward :  449.87542724609375\n",
      "place 210\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 210, 'x_pos_screen': 112, 'y_pos': 101}\n",
      "global_step :  88 , action :  3 action_prob :  [0.05506347119808197, 0.17301127314567566, 0.04001304879784584, 0.5822610259056091, 0.06770026683807373, 0.03870515152812004, 0.04324568435549736]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.8216552734375\n",
      "place 213\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 213, 'x_pos_screen': 112, 'y_pos': 97}\n",
      "global_step :  89 , action :  3 action_prob :  [0.05439199134707451, 0.17245344817638397, 0.03944229334592819, 0.5859199166297913, 0.06701147556304932, 0.03812000900506973, 0.042660921812057495]\n",
      "extrinsic_reward :  3 intrinsic_reward :  457.89459228515625\n",
      "place 216\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 216, 'x_pos_screen': 112, 'y_pos': 92}\n",
      "global_step :  90 , action :  3 action_prob :  [0.0549456961452961, 0.17291560769081116, 0.039906471967697144, 0.5829238891601562, 0.06756739318370819, 0.03860883414745331, 0.04313207045197487]\n",
      "extrinsic_reward :  3 intrinsic_reward :  445.9731140136719\n",
      "place 219\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 219, 'x_pos_screen': 112, 'y_pos': 87}\n",
      "global_step :  91 , action :  1 action_prob :  [0.054616931825876236, 0.17264480888843536, 0.03962993994355202, 0.584701418876648, 0.06723668426275253, 0.03831801563501358, 0.04285210371017456]\n",
      "extrinsic_reward :  3 intrinsic_reward :  446.91009521484375\n",
      "place 222\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 222, 'x_pos_screen': 112, 'y_pos': 82}\n",
      "global_step :  92 , action :  3 action_prob :  [0.05502422899007797, 0.1729591190814972, 0.039976079016923904, 0.582512617111206, 0.06765227764844894, 0.03866896033287048, 0.043206747621297836]\n",
      "extrinsic_reward :  3 intrinsic_reward :  446.46417236328125\n",
      "place 225\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 225, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  93 , action :  3 action_prob :  [0.055436182767152786, 0.17328812181949615, 0.04032553732395172, 0.5802863836288452, 0.0680708959698677, 0.03902935981750488, 0.043563585728406906]\n",
      "extrinsic_reward :  3 intrinsic_reward :  445.52618408203125\n",
      "place 228\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 228, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  94 , action :  3 action_prob :  [0.05518469586968422, 0.17309319972991943, 0.04011673852801323, 0.5816195607185364, 0.06782633811235428, 0.0388045608997345, 0.043354835361242294]\n",
      "extrinsic_reward :  3 intrinsic_reward :  459.12994384765625\n",
      "place 231\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 231, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  95 , action :  3 action_prob :  [0.055370040237903595, 0.17320950329303741, 0.04027896746993065, 0.5806389451026917, 0.06802292913198471, 0.03895365446805954, 0.043525952845811844]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.060302734375\n",
      "place 234\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 234, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  96 , action :  3 action_prob :  [0.05549226701259613, 0.17332404851913452, 0.04038234055042267, 0.5799577832221985, 0.06814835965633392, 0.03906317427754402, 0.043632086366415024]\n",
      "extrinsic_reward :  3 intrinsic_reward :  452.73614501953125\n",
      "place 237\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 237, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  97 , action :  3 action_prob :  [0.05491248890757561, 0.1728445142507553, 0.039889413863420486, 0.583114504814148, 0.06755606085062027, 0.03855535760521889, 0.043127622455358505]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.1728820800781\n",
      "place 240\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 240, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  98 , action :  0 action_prob :  [0.0556146577000618, 0.1734006106853485, 0.04048677906394005, 0.5793187022209167, 0.06827376782894135, 0.03916449472308159, 0.043740902096033096]\n",
      "extrinsic_reward :  3 intrinsic_reward :  444.870849609375\n",
      "place 243\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 243, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  99 , action :  1 action_prob :  [0.05524880811572075, 0.17311173677444458, 0.04017762467265129, 0.5812863111495972, 0.06790617853403091, 0.03884212300181389, 0.04342721775174141]\n",
      "extrinsic_reward :  2 intrinsic_reward :  449.9960632324219\n",
      "place 246\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 246, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  100 , action :  3 action_prob :  [0.055564604699611664, 0.1733541190624237, 0.04044490307569504, 0.5795932412147522, 0.06822578608989716, 0.039116546511650085, 0.043700795620679855]\n",
      "extrinsic_reward :  3 intrinsic_reward :  441.7859802246094\n",
      "place 249\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 249, 'x_pos_screen': 112, 'y_pos': 79}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  101 , action :  6 action_prob :  [0.055941108614206314, 0.17364652454853058, 0.040764544159173965, 0.577569842338562, 0.06860467791557312, 0.039448291063308716, 0.04402506723999977]\n",
      "extrinsic_reward :  3 intrinsic_reward :  442.518310546875\n",
      "place 252\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 252, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  102 , action :  2 action_prob :  [0.055630967020988464, 0.1734083890914917, 0.04050486534833908, 0.5792195200920105, 0.06830023974180222, 0.03917165473103523, 0.04376440495252609]\n",
      "extrinsic_reward :  3 intrinsic_reward :  454.26873779296875\n",
      "place 255\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 255, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  103 , action :  0 action_prob :  [0.05565628036856651, 0.17342279851436615, 0.04052228853106499, 0.5791031122207642, 0.06831934303045273, 0.03919543698430061, 0.04378078877925873]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.7576904296875\n",
      "place 258\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 258, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  104 , action :  3 action_prob :  [0.055736299604177475, 0.17349733412265778, 0.04059085622429848, 0.5786551237106323, 0.06840217113494873, 0.039266955107450485, 0.043851181864738464]\n",
      "extrinsic_reward :  3 intrinsic_reward :  448.1751708984375\n",
      "place 261\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 261, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  105 , action :  3 action_prob :  [0.0550708994269371, 0.17295671999454498, 0.04002297297120094, 0.5822747349739075, 0.06771951913833618, 0.0386870875954628, 0.04326804354786873]\n",
      "extrinsic_reward :  3 intrinsic_reward :  445.5510559082031\n",
      "place 264\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 264, 'x_pos_screen': 112, 'y_pos': 77}\n",
      "global_step :  106 , action :  3 action_prob :  [0.055516745895147324, 0.17332269251346588, 0.04039803892374039, 0.5798640251159668, 0.06816776096820831, 0.039081037044525146, 0.04364970326423645]\n",
      "extrinsic_reward :  3 intrinsic_reward :  446.5234375\n",
      "place 267\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 267, 'x_pos_screen': 112, 'y_pos': 75}\n",
      "global_step :  107 , action :  3 action_prob :  [0.05536288395524025, 0.17319601774215698, 0.04027019813656807, 0.5806909799575806, 0.06801602244377136, 0.038944143801927567, 0.043519821017980576]\n",
      "extrinsic_reward :  3 intrinsic_reward :  453.29718017578125\n",
      "place 270\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 270, 'x_pos_screen': 112, 'y_pos': 71}\n",
      "global_step :  108 , action :  3 action_prob :  [0.05532496050000191, 0.17317333817481995, 0.04023309797048569, 0.5809065699577332, 0.0679662749171257, 0.038919512182474136, 0.04347629472613335]\n",
      "extrinsic_reward :  3 intrinsic_reward :  440.646240234375\n",
      "place 273\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 273, 'x_pos_screen': 112, 'y_pos': 67}\n",
      "global_step :  109 , action :  3 action_prob :  [0.055586960166692734, 0.1733778864145279, 0.04045301675796509, 0.5795051455497742, 0.06822720915079117, 0.03915071859955788, 0.043699029833078384]\n",
      "extrinsic_reward :  3 intrinsic_reward :  439.82720947265625\n",
      "place 276\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 276, 'x_pos_screen': 112, 'y_pos': 62}\n",
      "global_step :  110 , action :  5 action_prob :  [0.055228885263204575, 0.17309671640396118, 0.040152642875909805, 0.5814216136932373, 0.06787069886922836, 0.03883523866534233, 0.04339420795440674]\n",
      "extrinsic_reward :  3 intrinsic_reward :  447.19415283203125\n",
      "place 279\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 279, 'x_pos_screen': 112, 'y_pos': 57}\n",
      "global_step :  111 , action :  3 action_prob :  [0.05501611530780792, 0.17292875051498413, 0.03996599093079567, 0.582595705986023, 0.0676381066441536, 0.03866049647331238, 0.043194834142923355]\n",
      "extrinsic_reward :  3 intrinsic_reward :  441.89666748046875\n",
      "place 282\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 282, 'x_pos_screen': 112, 'y_pos': 52}\n",
      "global_step :  112 , action :  3 action_prob :  [0.05513044819235802, 0.1730235368013382, 0.040063779801130295, 0.5819703340530396, 0.06775660812854767, 0.03876001015305519, 0.04329519718885422]\n",
      "extrinsic_reward :  3 intrinsic_reward :  445.6610107421875\n",
      "place 285\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 285, 'x_pos_screen': 112, 'y_pos': 47}\n",
      "global_step :  113 , action :  1 action_prob :  [0.05458901822566986, 0.1725875735282898, 0.03960057720541954, 0.5849177837371826, 0.06719322502613068, 0.038296643644571304, 0.04281510412693024]\n",
      "extrinsic_reward :  3 intrinsic_reward :  442.9622802734375\n",
      "place 288\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 288, 'x_pos_screen': 112, 'y_pos': 42}\n",
      "global_step :  114 , action :  1 action_prob :  [0.054799169301986694, 0.17276540398597717, 0.03977647051215172, 0.5837832689285278, 0.06739961355924606, 0.03848659247159958, 0.04298948869109154]\n",
      "extrinsic_reward :  3 intrinsic_reward :  444.74176025390625\n",
      "place 291\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 291, 'x_pos_screen': 112, 'y_pos': 37}\n",
      "global_step :  115 , action :  4 action_prob :  [0.05451928824186325, 0.1725093126296997, 0.03954266011714935, 0.5853210687637329, 0.0671202540397644, 0.038233064115047455, 0.042754337191581726]\n",
      "extrinsic_reward :  3 intrinsic_reward :  446.3843994140625\n",
      "place 294\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 294, 'x_pos_screen': 112, 'y_pos': 32}\n",
      "global_step :  116 , action :  5 action_prob :  [0.05469416826963425, 0.17267577350139618, 0.039681971073150635, 0.5843842029571533, 0.06727547198534012, 0.038405854254961014, 0.04288256913423538]\n",
      "extrinsic_reward :  3 intrinsic_reward :  419.5480041503906\n",
      "place 297\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 297, 'x_pos_screen': 112, 'y_pos': 27}\n",
      "global_step :  117 , action :  3 action_prob :  [0.054849766194820404, 0.17278876900672913, 0.039814434945583344, 0.5835569500923157, 0.06743350625038147, 0.038538914173841476, 0.0430176705121994]\n",
      "extrinsic_reward :  3 intrinsic_reward :  419.62664794921875\n",
      "place 300\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 300, 'x_pos_screen': 112, 'y_pos': 22}\n",
      "global_step :  118 , action :  3 action_prob :  [0.05441586300730705, 0.17242147028446198, 0.03945399075746536, 0.585895836353302, 0.06700656563043594, 0.038149282336235046, 0.04265693202614784]\n",
      "extrinsic_reward :  3 intrinsic_reward :  421.01483154296875\n",
      "place 303\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 303, 'x_pos_screen': 112, 'y_pos': 17}\n",
      "global_step :  119 , action :  3 action_prob :  [0.054484862834215164, 0.17249834537506104, 0.03950286656618118, 0.5855402946472168, 0.06705142557621002, 0.03823121264576912, 0.042690932750701904]\n",
      "extrinsic_reward :  2 intrinsic_reward :  426.13751220703125\n",
      "place 306\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 306, 'x_pos_screen': 112, 'y_pos': 12}\n",
      "global_step :  120 , action :  3 action_prob :  [0.054533641785383224, 0.17253044247627258, 0.03954539820551872, 0.5852779746055603, 0.06710627675056458, 0.03826801851391792, 0.04273821786046028]\n",
      "extrinsic_reward :  3 intrinsic_reward :  419.7349548339844\n",
      "place 309\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 309, 'x_pos_screen': 112, 'y_pos': 7}\n",
      "global_step :  121 , action :  3 action_prob :  [0.05436483025550842, 0.1723988652229309, 0.03939753770828247, 0.5862072110176086, 0.0669204518198967, 0.03813132643699646, 0.04257972538471222]\n",
      "extrinsic_reward :  3 intrinsic_reward :  407.9710693359375\n",
      "place 312\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 312, 'x_pos_screen': 112, 'y_pos': 2}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :  3 , global_step :  122\n",
      "global_step :  1 , action :  1 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.56402587890625\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  2 , action :  1 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.56402587890625\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  3 , action :  3 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  410.9014892578125\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  4 , action :  3 action_prob :  [0.05981843173503876, 0.17652776837348938, 0.0441044345498085, 0.5566869974136353, 0.07253562659025192, 0.04289211705327034, 0.047434646636247635]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.946533203125\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  5 , action :  1 action_prob :  [0.05979273095726967, 0.1765090972185135, 0.04408237338066101, 0.5568243265151978, 0.0725105032324791, 0.04286839812994003, 0.04741258919239044]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.946533203125\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  6 , action :  1 action_prob :  [0.05979273095726967, 0.1765090972185135, 0.04408237338066101, 0.5568243265151978, 0.0725105032324791, 0.04286839812994003, 0.04741258919239044]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.946533203125\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 79}\n",
      "global_step :  7 , action :  0 action_prob :  [0.05979273095726967, 0.1765090972185135, 0.04408237338066101, 0.5568243265151978, 0.0725105032324791, 0.04286839812994003, 0.04741258919239044]\n",
      "extrinsic_reward :  1 intrinsic_reward :  413.6903381347656\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 79}\n",
      "global_step :  8 , action :  6 action_prob :  [0.05979194492101669, 0.17650891840457916, 0.0440816767513752, 0.5568281412124634, 0.07250966876745224, 0.042867839336395264, 0.04741181805729866]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.7662048339844\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 79}\n",
      "global_step :  9 , action :  4 action_prob :  [0.0597960539162159, 0.17651185393333435, 0.044085193425416946, 0.5568062663078308, 0.072513647377491, 0.04287165030837059, 0.047415316104888916]\n",
      "extrinsic_reward :  0 intrinsic_reward :  410.32073974609375\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 79}\n",
      "global_step :  10 , action :  3 action_prob :  [0.05979946628212929, 0.17651450634002686, 0.044088032096624374, 0.5567883849143982, 0.07251665741205215, 0.04287505894899368, 0.047417931258678436]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.7061767578125\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 84}\n",
      "global_step :  11 , action :  1 action_prob :  [0.05983111262321472, 0.1765364110469818, 0.04411520063877106, 0.5566203594207764, 0.07254783809185028, 0.04290372133255005, 0.047445397824048996]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.68798828125\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 89}\n",
      "global_step :  12 , action :  2 action_prob :  [0.059782128781080246, 0.17650151252746582, 0.04407273605465889, 0.5568831562995911, 0.07249872386455536, 0.042859409004449844, 0.04740225151181221]\n",
      "extrinsic_reward :  0 intrinsic_reward :  413.0453186035156\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 93}\n",
      "global_step :  13 , action :  3 action_prob :  [0.059696294367313385, 0.17644479870796204, 0.043996941298246384, 0.5573437213897705, 0.07240986078977585, 0.04278464987874031, 0.04732367396354675]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.3920593261719\n",
      "place 43\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 43, 'x_pos_screen': 43, 'y_pos': 96}\n",
      "global_step :  14 , action :  1 action_prob :  [0.059714093804359436, 0.17645888030529022, 0.04401199519634247, 0.5572479963302612, 0.0724269449710846, 0.04280154034495354, 0.04733853042125702]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.7631530761719\n",
      "place 43\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 43, 'x_pos_screen': 43, 'y_pos': 99}\n",
      "global_step :  15 , action :  3 action_prob :  [0.05974277853965759, 0.17647862434387207, 0.04403705149888992, 0.5570942163467407, 0.07245605438947678, 0.042827144265174866, 0.04736414924263954]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.044921875\n",
      "place 44\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 44, 'x_pos_screen': 44, 'y_pos': 101}\n",
      "global_step :  16 , action :  4 action_prob :  [0.059714365750551224, 0.17645934224128723, 0.044011879712343216, 0.5572476387023926, 0.0724264457821846, 0.04280225932598114, 0.047338105738162994]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.1961975097656\n",
      "place 44\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 44, 'x_pos_screen': 44, 'y_pos': 102}\n",
      "global_step :  17 , action :  3 action_prob :  [0.05973837897181511, 0.1764756441116333, 0.044032830744981766, 0.5571191906929016, 0.07245080173015594, 0.04282353073358536, 0.04735962301492691]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.77130126953125\n",
      "place 45\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 45, 'x_pos_screen': 45, 'y_pos': 103}\n",
      "global_step :  18 , action :  1 action_prob :  [0.05975436791777611, 0.17648683488368988, 0.04404667392373085, 0.5570337176322937, 0.07246679067611694, 0.04283800348639488, 0.04737365245819092]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.27459716796875\n",
      "place 46\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 46, 'x_pos_screen': 46, 'y_pos': 103}\n",
      "global_step :  19 , action :  3 action_prob :  [0.05979393422603607, 0.1765131652355194, 0.04408162087202072, 0.5568210482597351, 0.07250770926475525, 0.04287263751029968, 0.0474097803235054]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.769775390625\n",
      "place 47\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 47, 'x_pos_screen': 47, 'y_pos': 102}\n",
      "global_step :  20 , action :  0 action_prob :  [0.059774283319711685, 0.1764993667602539, 0.04406454786658287, 0.5569264888763428, 0.07248786091804504, 0.04285499081015587, 0.0473923422396183]\n",
      "extrinsic_reward :  1 intrinsic_reward :  409.5466613769531\n",
      "place 48\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 48, 'x_pos_screen': 48, 'y_pos': 101}\n",
      "global_step :  21 , action :  0 action_prob :  [0.059771452099084854, 0.1764976680278778, 0.044061578810214996, 0.556943416595459, 0.07248391211032867, 0.04285316541790962, 0.04738881066441536]\n",
      "extrinsic_reward :  2 intrinsic_reward :  410.2262268066406\n",
      "place 50\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 50, 'x_pos_screen': 50, 'y_pos': 99}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  22 , action :  6 action_prob :  [0.05976485833525658, 0.17649368941783905, 0.04405582696199417, 0.5569781064987183, 0.07247728109359741, 0.04284738376736641, 0.04738292843103409]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.0460205078125\n",
      "place 51\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 51, 'x_pos_screen': 51, 'y_pos': 96}\n",
      "global_step :  23 , action :  6 action_prob :  [0.05981658399105072, 0.17652851343154907, 0.04410143941640854, 0.5567000508308411, 0.07253064215183258, 0.04289275035262108, 0.04743005335330963]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.88153076171875\n",
      "place 52\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 52, 'x_pos_screen': 52, 'y_pos': 93}\n",
      "global_step :  24 , action :  3 action_prob :  [0.05985487625002861, 0.17655512690544128, 0.04413506016135216, 0.5564936995506287, 0.0725698322057724, 0.042926911264657974, 0.04746450483798981]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.39239501953125\n",
      "place 53\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 53, 'x_pos_screen': 53, 'y_pos': 89}\n",
      "global_step :  25 , action :  6 action_prob :  [0.059804391115903854, 0.1765204519033432, 0.044090673327445984, 0.5567654967308044, 0.0725179985165596, 0.04288218170404434, 0.047418851405382156]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.36865234375\n",
      "place 54\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 54, 'x_pos_screen': 54, 'y_pos': 85}\n",
      "global_step :  26 , action :  2 action_prob :  [0.059796590358018875, 0.17651477456092834, 0.04408404231071472, 0.5568070411682129, 0.07251036912202835, 0.042875032871961594, 0.047412145882844925]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.48486328125\n",
      "place 55\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 55, 'x_pos_screen': 55, 'y_pos': 79}\n",
      "global_step :  27 , action :  1 action_prob :  [0.05974661186337471, 0.1764804869890213, 0.04404067248106003, 0.5570737719535828, 0.0724601298570633, 0.04283040016889572, 0.04736786335706711]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.02996826171875\n",
      "place 56\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 56, 'x_pos_screen': 56, 'y_pos': 84}\n",
      "global_step :  28 , action :  1 action_prob :  [0.05986738204956055, 0.1765626221895218, 0.044146522879600525, 0.5564262270927429, 0.07258342951536179, 0.04293730482459068, 0.04747661575675011]\n",
      "extrinsic_reward :  1 intrinsic_reward :  409.9290771484375\n",
      "place 57\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 57, 'x_pos_screen': 57, 'y_pos': 89}\n",
      "global_step :  29 , action :  1 action_prob :  [0.0598284937441349, 0.17653600871562958, 0.04411226138472557, 0.5566355586051941, 0.07254345715045929, 0.0429028682410717, 0.047441400587558746]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.1135559082031\n",
      "place 58\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 58, 'x_pos_screen': 58, 'y_pos': 93}\n",
      "global_step :  30 , action :  3 action_prob :  [0.05985899269580841, 0.1765572875738144, 0.044138822704553604, 0.5564718842506409, 0.07257427275180817, 0.0429302453994751, 0.04746850207448006]\n",
      "extrinsic_reward :  2 intrinsic_reward :  410.8411865234375\n",
      "place 60\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 60, 'x_pos_screen': 60, 'y_pos': 96}\n",
      "global_step :  31 , action :  3 action_prob :  [0.05983510985970497, 0.17654180526733398, 0.04411748796701431, 0.5566005110740662, 0.07254915684461594, 0.0429096482694149, 0.047446299344301224]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.7981262207031\n",
      "place 61\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 61, 'x_pos_screen': 61, 'y_pos': 99}\n",
      "global_step :  32 , action :  6 action_prob :  [0.0598628856241703, 0.1765604168176651, 0.04414178803563118, 0.5564519762992859, 0.07257743924856186, 0.04293422773480415, 0.047471240162849426]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.9804382324219\n",
      "place 63\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 63, 'x_pos_screen': 63, 'y_pos': 101}\n",
      "global_step :  33 , action :  3 action_prob :  [0.059849753975868225, 0.17655056715011597, 0.04413065314292908, 0.5565222501754761, 0.07256469875574112, 0.042921897023916245, 0.0474601611495018]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.48321533203125\n",
      "place 65\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 65, 'x_pos_screen': 65, 'y_pos': 102}\n",
      "global_step :  34 , action :  1 action_prob :  [0.05979161709547043, 0.17651087045669556, 0.044079478830099106, 0.5568350553512573, 0.07250484079122543, 0.042870670557022095, 0.04740738868713379]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.46160888671875\n",
      "place 66\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 66, 'x_pos_screen': 66, 'y_pos': 103}\n",
      "global_step :  35 , action :  3 action_prob :  [0.05980479717254639, 0.1765201985836029, 0.04409080743789673, 0.5567649006843567, 0.07251783460378647, 0.042882729321718216, 0.047418780624866486]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.7723388671875\n",
      "place 68\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 68, 'x_pos_screen': 68, 'y_pos': 103}\n",
      "global_step :  36 , action :  3 action_prob :  [0.059805870056152344, 0.17652101814746857, 0.04409163072705269, 0.5567594766616821, 0.07251869142055511, 0.04288383945822716, 0.04741951450705528]\n",
      "extrinsic_reward :  2 intrinsic_reward :  414.15679931640625\n",
      "place 70\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 70, 'x_pos_screen': 70, 'y_pos': 102}\n",
      "global_step :  37 , action :  3 action_prob :  [0.05984305217862129, 0.17654651403427124, 0.044124312698841095, 0.5565593242645264, 0.07255692034959793, 0.04291665554046631, 0.04745322838425636]\n",
      "extrinsic_reward :  1 intrinsic_reward :  414.27276611328125\n",
      "place 71\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 71, 'x_pos_screen': 71, 'y_pos': 101}\n",
      "global_step :  38 , action :  3 action_prob :  [0.05982016399502754, 0.17652925848960876, 0.04410530999302864, 0.5566802620887756, 0.07253552973270416, 0.04289482906460762, 0.047434575855731964]\n",
      "extrinsic_reward :  2 intrinsic_reward :  413.09710693359375\n",
      "place 73\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 73, 'x_pos_screen': 73, 'y_pos': 99}\n",
      "global_step :  39 , action :  3 action_prob :  [0.059814807027578354, 0.1765260547399521, 0.044100236147642136, 0.5567100644111633, 0.07252922654151917, 0.042890697717666626, 0.047428954392671585]\n",
      "extrinsic_reward :  1 intrinsic_reward :  413.1613464355469\n",
      "place 75\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 75, 'x_pos_screen': 75, 'y_pos': 96}\n",
      "global_step :  40 , action :  3 action_prob :  [0.05990742892026901, 0.17659005522727966, 0.04418083652853966, 0.5562139749526978, 0.07262285053730011, 0.042973536998033524, 0.04751136526465416]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.6997985839844\n",
      "place 77\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 77, 'x_pos_screen': 77, 'y_pos': 93}\n",
      "global_step :  41 , action :  4 action_prob :  [0.059914249926805496, 0.1765935719013214, 0.0441870242357254, 0.556178092956543, 0.0726301521062851, 0.04297893866896629, 0.04751797765493393]\n",
      "extrinsic_reward :  1 intrinsic_reward :  409.9381408691406\n",
      "place 78\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 78, 'x_pos_screen': 78, 'y_pos': 89}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  42 , action :  1 action_prob :  [0.05988256633281708, 0.17657169699668884, 0.044159796088933945, 0.5563463568687439, 0.07259893417358398, 0.04295012354850769, 0.04749048128724098]\n",
      "extrinsic_reward :  2 intrinsic_reward :  410.1383972167969\n",
      "place 80\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 80, 'x_pos_screen': 80, 'y_pos': 85}\n",
      "global_step :  43 , action :  5 action_prob :  [0.05993623286485672, 0.1766081154346466, 0.044206928461790085, 0.5560583472251892, 0.0726538598537445, 0.04299764335155487, 0.04753890633583069]\n",
      "extrinsic_reward :  2 intrinsic_reward :  409.13525390625\n",
      "place 82\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 82, 'x_pos_screen': 81, 'y_pos': 79}\n",
      "global_step :  44 , action :  3 action_prob :  [0.059920188039541245, 0.1765977144241333, 0.04419270157814026, 0.556144118309021, 0.07263742387294769, 0.04298340529203415, 0.04752439633011818]\n",
      "extrinsic_reward :  2 intrinsic_reward :  398.9155578613281\n",
      "place 84\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 84, 'x_pos_screen': 82, 'y_pos': 84}\n",
      "global_step :  45 , action :  3 action_prob :  [0.058746904134750366, 0.17578570544719696, 0.04317743331193924, 0.5624078512191772, 0.07145752012729645, 0.041935909539461136, 0.04648871347308159]\n",
      "extrinsic_reward :  1 intrinsic_reward :  395.6454772949219\n",
      "place 85\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 85, 'x_pos_screen': 82, 'y_pos': 89}\n",
      "global_step :  46 , action :  4 action_prob :  [0.05971109867095947, 0.17644809186458588, 0.04402543976902962, 0.5572132468223572, 0.07245798408985138, 0.042776040732860565, 0.047368139028549194]\n",
      "extrinsic_reward :  2 intrinsic_reward :  419.9714660644531\n",
      "place 87\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 87, 'x_pos_screen': 83, 'y_pos': 93}\n",
      "global_step :  47 , action :  3 action_prob :  [0.059937670826911926, 0.1766001135110855, 0.04421944171190262, 0.5560199618339539, 0.0726773738861084, 0.04298453778028488, 0.04756086692214012]\n",
      "extrinsic_reward :  2 intrinsic_reward :  407.06707763671875\n",
      "place 89\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 89, 'x_pos_screen': 84, 'y_pos': 96}\n",
      "global_step :  48 , action :  3 action_prob :  [0.05914286896586418, 0.17604972422122955, 0.04353078827261925, 0.5602655410766602, 0.07187918573617935, 0.042272672057151794, 0.04685928300023079]\n",
      "extrinsic_reward :  2 intrinsic_reward :  426.662353515625\n",
      "place 91\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 91, 'x_pos_screen': 85, 'y_pos': 99}\n",
      "global_step :  49 , action :  1 action_prob :  [0.06030048057436943, 0.17684654891490936, 0.04453331604599953, 0.554090678691864, 0.07303780317306519, 0.04331133887171745, 0.04787973314523697]\n",
      "extrinsic_reward :  1 intrinsic_reward :  407.2264099121094\n",
      "place 92\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 92, 'x_pos_screen': 85, 'y_pos': 102}\n",
      "global_step :  50 , action :  3 action_prob :  [0.059829145669937134, 0.1765230894088745, 0.0441230908036232, 0.5566125512123108, 0.0725625604391098, 0.042889147996902466, 0.047460392117500305]\n",
      "extrinsic_reward :  2 intrinsic_reward :  419.18316650390625\n",
      "place 94\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 94, 'x_pos_screen': 86, 'y_pos': 104}\n",
      "global_step :  51 , action :  1 action_prob :  [0.060714904218912125, 0.17711345851421356, 0.04489247500896454, 0.5519028902053833, 0.07344792783260345, 0.04368503764271736, 0.04824327677488327]\n",
      "extrinsic_reward :  2 intrinsic_reward :  416.5826416015625\n",
      "place 96\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 96, 'x_pos_screen': 87, 'y_pos': 105}\n",
      "global_step :  52 , action :  0 action_prob :  [0.05957076698541641, 0.17634959518909454, 0.04389897361397743, 0.5579855442047119, 0.07230401039123535, 0.0426592156291008, 0.04723190888762474]\n",
      "extrinsic_reward :  2 intrinsic_reward :  396.02728271484375\n",
      "place 98\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 98, 'x_pos_screen': 88, 'y_pos': 106}\n",
      "global_step :  53 , action :  3 action_prob :  [0.0585295706987381, 0.1756187230348587, 0.042996782809495926, 0.563561201095581, 0.07125125825405121, 0.04173235595226288, 0.04631008952856064]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.1927490234375\n",
      "place 99\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 99, 'x_pos_screen': 88, 'y_pos': 106}\n",
      "global_step :  54 , action :  1 action_prob :  [0.059035804122686386, 0.1759733408689499, 0.04344625771045685, 0.5608083009719849, 0.0717887282371521, 0.04216659069061279, 0.04678097739815712]\n",
      "extrinsic_reward :  2 intrinsic_reward :  420.44427490234375\n",
      "place 101\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 101, 'x_pos_screen': 89, 'y_pos': 105}\n",
      "global_step :  55 , action :  3 action_prob :  [0.05928047373890877, 0.17614802718162537, 0.043654151260852814, 0.5595112442970276, 0.07202617079019547, 0.04239168018102646, 0.046988241374492645]\n",
      "extrinsic_reward :  2 intrinsic_reward :  416.3244323730469\n",
      "place 103\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 103, 'x_pos_screen': 90, 'y_pos': 104}\n",
      "global_step :  56 , action :  5 action_prob :  [0.058374643325805664, 0.17551204562187195, 0.042867470532655716, 0.5643675327301025, 0.07110537588596344, 0.04159087687730789, 0.04618204012513161]\n",
      "extrinsic_reward :  2 intrinsic_reward :  424.0585021972656\n",
      "place 105\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 105, 'x_pos_screen': 91, 'y_pos': 103}\n",
      "global_step :  57 , action :  6 action_prob :  [0.059606004506349564, 0.17637380957603455, 0.04393700882792473, 0.5577703714370728, 0.07235383242368698, 0.04268210008740425, 0.04727683588862419]\n",
      "extrinsic_reward :  1 intrinsic_reward :  416.59063720703125\n",
      "place 106\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 106, 'x_pos_screen': 91, 'y_pos': 100}\n",
      "global_step :  58 , action :  3 action_prob :  [0.05921236053109169, 0.17610019445419312, 0.043592244386672974, 0.559889018535614, 0.07194969058036804, 0.042335301637649536, 0.0469212532043457]\n",
      "extrinsic_reward :  2 intrinsic_reward :  421.85260009765625\n",
      "place 108\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 108, 'x_pos_screen': 92, 'y_pos': 98}\n",
      "global_step :  59 , action :  3 action_prob :  [0.059990834444761276, 0.1766270101070404, 0.0442688912153244, 0.5557386875152588, 0.07273425906896591, 0.04302782565355301, 0.04761253297328949]\n",
      "extrinsic_reward :  1 intrinsic_reward :  423.948974609375\n",
      "place 110\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 110, 'x_pos_screen': 93, 'y_pos': 95}\n",
      "global_step :  60 , action :  5 action_prob :  [0.058789126574993134, 0.1758088618516922, 0.043223921209573746, 0.5621510744094849, 0.07152140885591507, 0.04196024686098099, 0.046545371413230896]\n",
      "extrinsic_reward :  2 intrinsic_reward :  416.1866149902344\n",
      "place 112\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 112, 'x_pos_screen': 94, 'y_pos': 91}\n",
      "global_step :  61 , action :  3 action_prob :  [0.058068156242370605, 0.17528973519802094, 0.04259982332587242, 0.5660273432731628, 0.0707869753241539, 0.04132310301065445, 0.045904915779829025]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.9749755859375\n",
      "place 113\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 113, 'x_pos_screen': 94, 'y_pos': 86}\n",
      "global_step :  62 , action :  3 action_prob :  [0.058351073414087296, 0.17549099028110504, 0.04285375773906708, 0.564473032951355, 0.07109634578227997, 0.041559625416994095, 0.04617518186569214]\n",
      "extrinsic_reward :  2 intrinsic_reward :  435.295654296875\n",
      "place 115\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 115, 'x_pos_screen': 95, 'y_pos': 81}\n",
      "global_step :  63 , action :  0 action_prob :  [0.05861629545688629, 0.17568108439445496, 0.043079692870378494, 0.5630635023117065, 0.07135744392871857, 0.04179920256137848, 0.04640280455350876]\n",
      "extrinsic_reward :  2 intrinsic_reward :  423.0022277832031\n",
      "place 117\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 117, 'x_pos_screen': 96, 'y_pos': 79}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  64 , action :  0 action_prob :  [0.05790982022881508, 0.17516817152500153, 0.04246940836310387, 0.5668599009513855, 0.07064081728458405, 0.04117347300052643, 0.04577839374542236]\n",
      "extrinsic_reward :  1 intrinsic_reward :  435.24664306640625\n",
      "place 118\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 118, 'x_pos_screen': 96, 'y_pos': 79}\n",
      "global_step :  65 , action :  4 action_prob :  [0.05885159224271774, 0.1758505254983902, 0.04328231140971184, 0.5618041753768921, 0.07159284502267838, 0.042009416967630386, 0.04660908132791519]\n",
      "extrinsic_reward :  2 intrinsic_reward :  417.59832763671875\n",
      "place 120\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 120, 'x_pos_screen': 97, 'y_pos': 79}\n",
      "global_step :  66 , action :  3 action_prob :  [0.05846035107970238, 0.17557144165039062, 0.04294005408883095, 0.5639169216156006, 0.0711888000369072, 0.04166701063513756, 0.04625546187162399]\n",
      "extrinsic_reward :  2 intrinsic_reward :  430.1199645996094\n",
      "place 122\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 122, 'x_pos_screen': 98, 'y_pos': 84}\n",
      "global_step :  67 , action :  3 action_prob :  [0.05907491222023964, 0.17600704729557037, 0.04347256198525429, 0.5606222748756409, 0.07181074470281601, 0.04221292957663536, 0.04679953679442406]\n",
      "extrinsic_reward :  1 intrinsic_reward :  419.6490478515625\n",
      "place 123\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 123, 'x_pos_screen': 98, 'y_pos': 89}\n",
      "global_step :  68 , action :  3 action_prob :  [0.05811428651213646, 0.17532142996788025, 0.04264485463500023, 0.5657596588134766, 0.07084659487009048, 0.04135590419173241, 0.04595723748207092]\n",
      "extrinsic_reward :  2 intrinsic_reward :  413.8828125\n",
      "place 125\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 125, 'x_pos_screen': 99, 'y_pos': 93}\n",
      "global_step :  69 , action :  3 action_prob :  [0.05736735090613365, 0.17476899921894073, 0.04199652373790741, 0.569799542427063, 0.07007677853107452, 0.04070139303803444, 0.04528949037194252]\n",
      "extrinsic_reward :  2 intrinsic_reward :  428.0151062011719\n",
      "place 127\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 127, 'x_pos_screen': 100, 'y_pos': 96}\n",
      "global_step :  70 , action :  3 action_prob :  [0.05764089152216911, 0.1749708652496338, 0.042243752628564835, 0.5682823061943054, 0.07038044184446335, 0.04092846438288689, 0.04555317759513855]\n",
      "extrinsic_reward :  2 intrinsic_reward :  438.344970703125\n",
      "place 129\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 129, 'x_pos_screen': 101, 'y_pos': 99}\n",
      "global_step :  71 , action :  3 action_prob :  [0.057844121009111404, 0.17512275278568268, 0.0424143522977829, 0.5672032833099365, 0.07057780772447586, 0.04111356660723686, 0.0457240492105484]\n",
      "extrinsic_reward :  1 intrinsic_reward :  429.47406005859375\n",
      "place 130\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 130, 'x_pos_screen': 101, 'y_pos': 102}\n",
      "global_step :  72 , action :  3 action_prob :  [0.05723721906542778, 0.17467100918293, 0.04188957437872887, 0.5704802870750427, 0.06995627284049988, 0.04058057814836502, 0.04518507793545723]\n",
      "extrinsic_reward :  2 intrinsic_reward :  436.05450439453125\n",
      "place 132\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 132, 'x_pos_screen': 102, 'y_pos': 104}\n",
      "global_step :  73 , action :  3 action_prob :  [0.0581902377307415, 0.1753670573234558, 0.042715560644865036, 0.5653451681137085, 0.0709327757358551, 0.04141496121883392, 0.04603422060608864]\n",
      "extrinsic_reward :  2 intrinsic_reward :  424.8062438964844\n",
      "place 134\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 134, 'x_pos_screen': 103, 'y_pos': 105}\n",
      "global_step :  74 , action :  3 action_prob :  [0.05783349275588989, 0.1751120239496231, 0.04240154102444649, 0.567279040813446, 0.07055824249982834, 0.041108399629592896, 0.04570721462368965]\n",
      "extrinsic_reward :  2 intrinsic_reward :  430.30511474609375\n",
      "place 136\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 136, 'x_pos_screen': 104, 'y_pos': 106}\n",
      "global_step :  75 , action :  3 action_prob :  [0.0584108829498291, 0.17552608251571655, 0.04290318116545677, 0.564172625541687, 0.07114974409341812, 0.041614141315221786, 0.04622326418757439]\n",
      "extrinsic_reward :  1 intrinsic_reward :  425.21478271484375\n",
      "place 137\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 137, 'x_pos_screen': 104, 'y_pos': 106}\n",
      "global_step :  76 , action :  3 action_prob :  [0.05731520429253578, 0.17473365366458893, 0.04195579141378403, 0.5700567364692688, 0.07003555446863174, 0.040649350732564926, 0.04525372013449669]\n",
      "extrinsic_reward :  2 intrinsic_reward :  421.700927734375\n",
      "place 139\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 139, 'x_pos_screen': 105, 'y_pos': 105}\n",
      "global_step :  77 , action :  3 action_prob :  [0.05675913393497467, 0.17431578040122986, 0.04147272929549217, 0.5730741024017334, 0.0694570317864418, 0.04016764089465141, 0.044753603637218475]\n",
      "extrinsic_reward :  2 intrinsic_reward :  428.3619384765625\n",
      "place 141\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 141, 'x_pos_screen': 106, 'y_pos': 104}\n",
      "global_step :  78 , action :  5 action_prob :  [0.05679072067141533, 0.17433381080627441, 0.04151017591357231, 0.5728700757026672, 0.06951186805963516, 0.040181342512369156, 0.04480202496051788]\n",
      "extrinsic_reward :  2 intrinsic_reward :  431.11907958984375\n",
      "place 143\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 297, 'world': 1, 'x_pos': 143, 'x_pos_screen': 107, 'y_pos': 103}\n",
      "global_step :  79 , action :  3 action_prob :  [0.05714001506567001, 0.174595445394516, 0.04180854186415672, 0.5709958076477051, 0.06986440718173981, 0.040489017963409424, 0.04510676488280296]\n",
      "extrinsic_reward :  0 intrinsic_reward :  425.9758605957031\n",
      "place 144\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 144, 'x_pos_screen': 107, 'y_pos': 100}\n",
      "global_step :  80 , action :  1 action_prob :  [0.05668997764587402, 0.17424552142620087, 0.041422367095947266, 0.5734338760375977, 0.06940505653619766, 0.04009227082133293, 0.04471083730459213]\n",
      "extrinsic_reward :  2 intrinsic_reward :  441.845947265625\n",
      "place 146\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 146, 'x_pos_screen': 108, 'y_pos': 98}\n",
      "global_step :  81 , action :  1 action_prob :  [0.057402145117521286, 0.17478439211845398, 0.04203511029481888, 0.5695905089378357, 0.07013143599033356, 0.04071732237935066, 0.04533907771110535]\n",
      "extrinsic_reward :  2 intrinsic_reward :  420.64385986328125\n",
      "place 148\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 148, 'x_pos_screen': 109, 'y_pos': 95}\n",
      "global_step :  82 , action :  1 action_prob :  [0.05713263899087906, 0.174586683511734, 0.04179691895842552, 0.571060836315155, 0.06984493136405945, 0.040487904101610184, 0.04509011656045914]\n",
      "extrinsic_reward :  2 intrinsic_reward :  434.0046081542969\n",
      "place 150\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 150, 'x_pos_screen': 110, 'y_pos': 91}\n",
      "global_step :  83 , action :  3 action_prob :  [0.05770100653171539, 0.17500261962413788, 0.04229229688644409, 0.567987859249115, 0.0704338550567627, 0.04098060354590416, 0.045601774007081985]\n",
      "extrinsic_reward :  1 intrinsic_reward :  421.98736572265625\n",
      "place 151\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 151, 'x_pos_screen': 110, 'y_pos': 86}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  84 , action :  3 action_prob :  [0.05674042925238609, 0.1742851436138153, 0.04146696999669075, 0.5731550455093384, 0.06945998221635818, 0.040133386850357056, 0.044759009033441544]\n",
      "extrinsic_reward :  2 intrinsic_reward :  414.9064636230469\n",
      "place 153\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 153, 'x_pos_screen': 111, 'y_pos': 81}\n",
      "global_step :  85 , action :  1 action_prob :  [0.05636643245816231, 0.17399361729621887, 0.04114091023802757, 0.5752026438713074, 0.0690656304359436, 0.03981093317270279, 0.044419895857572556]\n",
      "extrinsic_reward :  2 intrinsic_reward :  424.3379821777344\n",
      "place 155\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 155, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  86 , action :  6 action_prob :  [0.056192975491285324, 0.17385348677635193, 0.041002366691827774, 0.5761081576347351, 0.06890958547592163, 0.039647042751312256, 0.04428636655211449]\n",
      "extrinsic_reward :  2 intrinsic_reward :  432.696533203125\n",
      "place 157\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 157, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  87 , action :  2 action_prob :  [0.0564790740609169, 0.1740751713514328, 0.04124366119503975, 0.5745751857757568, 0.06919470429420471, 0.039900846779346466, 0.044531334191560745]\n",
      "extrinsic_reward :  1 intrinsic_reward :  438.95196533203125\n",
      "place 158\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 158, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  88 , action :  1 action_prob :  [0.05660617724061012, 0.1741715669631958, 0.04135242849588394, 0.5738916993141174, 0.06932348012924194, 0.04001195728778839, 0.04464271292090416]\n",
      "extrinsic_reward :  2 intrinsic_reward :  430.08197021484375\n",
      "place 160\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 160, 'x_pos_screen': 112, 'y_pos': 84}\n",
      "global_step :  89 , action :  3 action_prob :  [0.05654742568731308, 0.17412704229354858, 0.041298117488622665, 0.5742238163948059, 0.06925460696220398, 0.039966076612472534, 0.04458284750580788]\n",
      "extrinsic_reward :  2 intrinsic_reward :  433.16864013671875\n",
      "place 162\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 162, 'x_pos_screen': 112, 'y_pos': 89}\n",
      "global_step :  90 , action :  3 action_prob :  [0.05615991726517677, 0.17382727563381195, 0.04097001999616623, 0.5763025283813477, 0.06886818259954453, 0.03962051495909691, 0.04425151273608208]\n",
      "extrinsic_reward :  2 intrinsic_reward :  437.6968078613281\n",
      "place 164\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 164, 'x_pos_screen': 112, 'y_pos': 93}\n",
      "global_step :  91 , action :  3 action_prob :  [0.05559821054339409, 0.17338280379772186, 0.04049099609255791, 0.5793419480323792, 0.06829632073640823, 0.039128270000219345, 0.043761447072029114]\n",
      "extrinsic_reward :  1 intrinsic_reward :  435.2099609375\n",
      "place 165\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 165, 'x_pos_screen': 112, 'y_pos': 96}\n",
      "global_step :  92 , action :  3 action_prob :  [0.05551649630069733, 0.17332501709461212, 0.040415436029434204, 0.5797979831695557, 0.0682009607553482, 0.03906551003456116, 0.043678589165210724]\n",
      "extrinsic_reward :  2 intrinsic_reward :  448.116943359375\n",
      "place 167\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 167, 'x_pos_screen': 112, 'y_pos': 99}\n",
      "global_step :  93 , action :  2 action_prob :  [0.056040748953819275, 0.17373625934123993, 0.040866877883672714, 0.5769505500793457, 0.06874259561300278, 0.03951946645975113, 0.0441434271633625]\n",
      "extrinsic_reward :  2 intrinsic_reward :  444.52703857421875\n",
      "place 169\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 169, 'x_pos_screen': 112, 'y_pos': 102}\n",
      "global_step :  94 , action :  3 action_prob :  [0.05619736760854721, 0.17386764287948608, 0.0409979447722435, 0.5761024951934814, 0.06889760494232178, 0.039661526679992676, 0.04427534341812134]\n",
      "extrinsic_reward :  2 intrinsic_reward :  428.16632080078125\n",
      "place 171\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 171, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  95 , action :  3 action_prob :  [0.055668968707323074, 0.17345069348812103, 0.04054548963904381, 0.5789690613746643, 0.06835461407899857, 0.03920111060142517, 0.04381003975868225]\n",
      "extrinsic_reward :  1 intrinsic_reward :  446.81817626953125\n",
      "place 172\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 172, 'x_pos_screen': 112, 'y_pos': 105}\n",
      "global_step :  96 , action :  3 action_prob :  [0.05549878254532814, 0.17331348359584808, 0.04040278121829033, 0.5798837542533875, 0.0681857317686081, 0.039050228893756866, 0.043665334582328796]\n",
      "extrinsic_reward :  2 intrinsic_reward :  442.751953125\n",
      "place 174\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 174, 'x_pos_screen': 112, 'y_pos': 106}\n",
      "global_step :  97 , action :  1 action_prob :  [0.05503479763865471, 0.17295530438423157, 0.039999719709157944, 0.5824099183082581, 0.06769684702157974, 0.03865739330649376, 0.043245986104011536]\n",
      "extrinsic_reward :  2 intrinsic_reward :  444.6253356933594\n",
      "place 176\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 176, 'x_pos_screen': 112, 'y_pos': 106}\n",
      "global_step :  98 , action :  1 action_prob :  [0.055541057139635086, 0.1733546257019043, 0.04043194651603699, 0.5796709656715393, 0.06821581721305847, 0.039095647633075714, 0.04368999972939491]\n",
      "extrinsic_reward :  2 intrinsic_reward :  450.3613586425781\n",
      "place 178\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 296, 'world': 1, 'x_pos': 178, 'x_pos_screen': 112, 'y_pos': 105}\n",
      "global_step :  99 , action :  3 action_prob :  [0.055119551718235016, 0.1730174422264099, 0.040076956152915955, 0.5819395184516907, 0.06779345124959946, 0.03872278705239296, 0.043330296874046326]\n",
      "extrinsic_reward :  0 intrinsic_reward :  445.40625\n",
      "place 179\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 179, 'x_pos_screen': 112, 'y_pos': 104}\n",
      "global_step :  100 , action :  3 action_prob :  [0.05469432473182678, 0.1726679503917694, 0.039715081453323364, 0.5842538475990295, 0.06735684722661972, 0.03835330903530121, 0.04295860230922699]\n",
      "extrinsic_reward :  2 intrinsic_reward :  454.30047607421875\n",
      "place 181\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 181, 'x_pos_screen': 112, 'y_pos': 103}\n",
      "global_step :  101 , action :  3 action_prob :  [0.05512409284710884, 0.17302343249320984, 0.04007982835173607, 0.581917405128479, 0.06779413670301437, 0.03873143345117569, 0.04332966357469559]\n",
      "extrinsic_reward :  2 intrinsic_reward :  449.0382080078125\n",
      "place 183\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 183, 'x_pos_screen': 112, 'y_pos': 100}\n",
      "global_step :  102 , action :  3 action_prob :  [0.055134180933237076, 0.1730380803346634, 0.04008510336279869, 0.5818675756454468, 0.0677976906299591, 0.038745250552892685, 0.04333210363984108]\n",
      "extrinsic_reward :  2 intrinsic_reward :  446.62298583984375\n",
      "place 185\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 185, 'x_pos_screen': 112, 'y_pos': 98}\n",
      "global_step :  103 , action :  3 action_prob :  [0.055345650762319565, 0.1732153445482254, 0.04026376083493233, 0.5807176232337952, 0.06801138818264008, 0.03893214836716652, 0.043514057993888855]\n",
      "extrinsic_reward :  1 intrinsic_reward :  446.563232421875\n",
      "place 186\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 186, 'x_pos_screen': 112, 'y_pos': 95}\n",
      "global_step :  104 , action :  1 action_prob :  [0.055104125291109085, 0.17302176356315613, 0.040056366473436356, 0.5820315480232239, 0.06776230037212372, 0.03872276842594147, 0.04330119863152504]\n",
      "extrinsic_reward :  2 intrinsic_reward :  448.0907897949219\n",
      "place 188\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 188, 'x_pos_screen': 112, 'y_pos': 91}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  105 , action :  1 action_prob :  [0.054547686129808426, 0.1725568324327469, 0.039587266743183136, 0.5850530862808228, 0.06719900667667389, 0.038232848048210144, 0.04282330349087715]\n",
      "extrinsic_reward :  2 intrinsic_reward :  445.72161865234375\n",
      "place 190\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 190, 'x_pos_screen': 112, 'y_pos': 86}\n",
      "global_step :  106 , action :  1 action_prob :  [0.0545901320874691, 0.17261511087417603, 0.03961312770843506, 0.5848311185836792, 0.06722328811883926, 0.038286641240119934, 0.04284051060676575]\n",
      "extrinsic_reward :  2 intrinsic_reward :  449.368408203125\n",
      "place 192\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 192, 'x_pos_screen': 112, 'y_pos': 81}\n",
      "global_step :  107 , action :  4 action_prob :  [0.0550152063369751, 0.1729467362165451, 0.03997647389769554, 0.5825361013412476, 0.06766002625226974, 0.038651879876852036, 0.04321350157260895]\n",
      "extrinsic_reward :  1 intrinsic_reward :  448.4924621582031\n",
      "place 193\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 193, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  108 , action :  4 action_prob :  [0.05453953519463539, 0.1725626289844513, 0.03957575932145119, 0.5851003527641296, 0.06718091666698456, 0.03823409602046013, 0.04280669987201691]\n",
      "extrinsic_reward :  2 intrinsic_reward :  443.12176513671875\n",
      "place 195\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 195, 'x_pos_screen': 112, 'y_pos': 84}\n",
      "global_step :  109 , action :  3 action_prob :  [0.05492771416902542, 0.17288608849048615, 0.039904698729515076, 0.5829882621765137, 0.06757628917694092, 0.03857524320483208, 0.04314160719513893]\n",
      "extrinsic_reward :  2 intrinsic_reward :  449.61248779296875\n",
      "place 197\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 197, 'x_pos_screen': 112, 'y_pos': 89}\n",
      "global_step :  110 , action :  2 action_prob :  [0.05471048131585121, 0.17270775139331818, 0.03971640020608902, 0.5841837525367737, 0.06734585762023926, 0.0383903943002224, 0.042945392429828644]\n",
      "extrinsic_reward :  2 intrinsic_reward :  455.6268615722656\n",
      "place 199\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 199, 'x_pos_screen': 112, 'y_pos': 93}\n",
      "global_step :  111 , action :  3 action_prob :  [0.05472381040453911, 0.17273780703544617, 0.03972190245985985, 0.5841083526611328, 0.0673496276140213, 0.038412053138017654, 0.04294639825820923]\n",
      "extrinsic_reward :  1 intrinsic_reward :  453.54376220703125\n",
      "place 200\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 200, 'x_pos_screen': 112, 'y_pos': 97}\n",
      "global_step :  112 , action :  4 action_prob :  [0.05495012179017067, 0.17291943728923798, 0.03991736099123955, 0.5828724503517151, 0.06758696585893631, 0.038604333996772766, 0.04314938932657242]\n",
      "extrinsic_reward :  2 intrinsic_reward :  444.36517333984375\n",
      "place 202\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 202, 'x_pos_screen': 112, 'y_pos': 101}\n",
      "global_step :  113 , action :  3 action_prob :  [0.05447512865066528, 0.17252975702285767, 0.03951476141810417, 0.5854490995407104, 0.06710312515497208, 0.038189928978681564, 0.04273821413516998]\n",
      "extrinsic_reward :  2 intrinsic_reward :  448.4830322265625\n",
      "place 204\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 204, 'x_pos_screen': 112, 'y_pos': 103}\n",
      "global_step :  114 , action :  3 action_prob :  [0.05432543158531189, 0.17239554226398468, 0.03938908874988556, 0.5862717032432556, 0.06695093959569931, 0.038057517260313034, 0.04260978102684021]\n",
      "extrinsic_reward :  2 intrinsic_reward :  449.9172668457031\n",
      "place 206\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 206, 'x_pos_screen': 112, 'y_pos': 106}\n",
      "global_step :  115 , action :  3 action_prob :  [0.05468800291419029, 0.17270833253860474, 0.03969201445579529, 0.5843029022216797, 0.06731189787387848, 0.03838272765278816, 0.04291405528783798]\n",
      "extrinsic_reward :  1 intrinsic_reward :  450.27398681640625\n",
      "place 207\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 207, 'x_pos_screen': 112, 'y_pos': 107}\n",
      "global_step :  116 , action :  5 action_prob :  [0.054874129593372345, 0.17286747694015503, 0.03984176367521286, 0.5833133459091187, 0.06748726963996887, 0.03855416923761368, 0.04306182265281677]\n",
      "extrinsic_reward :  2 intrinsic_reward :  452.7882080078125\n",
      "place 209\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 209, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  117 , action :  0 action_prob :  [0.05495346710085869, 0.1729256808757782, 0.03991701081395149, 0.5828620791435242, 0.06758388131856918, 0.03861187398433685, 0.04314607381820679]\n",
      "extrinsic_reward :  2 intrinsic_reward :  448.029541015625\n",
      "place 211\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 211, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  118 , action :  3 action_prob :  [0.05487103387713432, 0.1728648841381073, 0.03984609991312027, 0.5833046436309814, 0.06749790161848068, 0.03854351118206978, 0.043071918189525604]\n",
      "extrinsic_reward :  2 intrinsic_reward :  456.48663330078125\n",
      "place 213\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 295, 'world': 1, 'x_pos': 213, 'x_pos_screen': 112, 'y_pos': 109}\n",
      "global_step :  119 , action :  3 action_prob :  [0.05488069728016853, 0.1728576272726059, 0.039853211492300034, 0.5832753777503967, 0.06750469654798508, 0.03854947164654732, 0.04307897016406059]\n",
      "extrinsic_reward :  0 intrinsic_reward :  455.98809814453125\n",
      "place 214\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 214, 'x_pos_screen': 112, 'y_pos': 108}\n",
      "global_step :  120 , action :  1 action_prob :  [0.05498089641332626, 0.17294463515281677, 0.03993690386414528, 0.5827304720878601, 0.06760429590940475, 0.038639575242996216, 0.0431632399559021]\n",
      "extrinsic_reward :  2 intrinsic_reward :  446.36187744140625\n",
      "place 216\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 216, 'x_pos_screen': 112, 'y_pos': 107}\n",
      "global_step :  121 , action :  3 action_prob :  [0.05513574182987213, 0.173069566488266, 0.04006524384021759, 0.5819030404090881, 0.06775602698326111, 0.03877827525138855, 0.043292123824357986]\n",
      "extrinsic_reward :  2 intrinsic_reward :  448.509765625\n",
      "place 218\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 218, 'x_pos_screen': 112, 'y_pos': 105}\n",
      "global_step :  122 , action :  1 action_prob :  [0.05457816272974014, 0.1726200133562088, 0.039595868438482285, 0.5849066972732544, 0.06719572842121124, 0.03828726336359978, 0.04281619191169739]\n",
      "extrinsic_reward :  2 intrinsic_reward :  450.0329895019531\n",
      "place 220\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 220, 'x_pos_screen': 112, 'y_pos': 103}\n",
      "global_step :  123 , action :  3 action_prob :  [0.05440806224942207, 0.17247258126735687, 0.039450351148843765, 0.5858450531959534, 0.06701817363500595, 0.038140036165714264, 0.042665816843509674]\n",
      "extrinsic_reward :  1 intrinsic_reward :  452.3450927734375\n",
      "place 221\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 221, 'x_pos_screen': 112, 'y_pos': 100}\n",
      "global_step :  124 , action :  3 action_prob :  [0.05512704700231552, 0.17304591834545135, 0.04006568714976311, 0.581942081451416, 0.06776156276464462, 0.038758546113967896, 0.0432991124689579]\n",
      "extrinsic_reward :  2 intrinsic_reward :  444.1775207519531\n",
      "place 223\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 223, 'x_pos_screen': 112, 'y_pos': 97}\n",
      "global_step :  125 , action :  3 action_prob :  [0.05523082986474037, 0.17313042283058167, 0.04015221819281578, 0.581384539604187, 0.06786483526229858, 0.03884992003440857, 0.04338723048567772]\n",
      "extrinsic_reward :  2 intrinsic_reward :  447.960205078125\n",
      "place 225\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 225, 'x_pos_screen': 112, 'y_pos': 93}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  126 , action :  1 action_prob :  [0.05523906275629997, 0.17314782738685608, 0.04015856981277466, 0.5813271999359131, 0.06787451356649399, 0.038857731968164444, 0.043395042419433594]\n",
      "extrinsic_reward :  2 intrinsic_reward :  445.3746337890625\n",
      "place 227\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 227, 'x_pos_screen': 112, 'y_pos': 88}\n",
      "global_step :  127 , action :  3 action_prob :  [0.05516177415847778, 0.1730823963880539, 0.04009426385164261, 0.5817447304725647, 0.06779816746711731, 0.03878873214125633, 0.04333003982901573]\n",
      "extrinsic_reward :  1 intrinsic_reward :  460.3038024902344\n",
      "place 228\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 228, 'x_pos_screen': 112, 'y_pos': 83}\n",
      "global_step :  128 , action :  1 action_prob :  [0.05533274635672569, 0.17318947613239288, 0.0402434766292572, 0.5808427929878235, 0.06797784566879272, 0.038927242159843445, 0.04348641633987427]\n",
      "extrinsic_reward :  2 intrinsic_reward :  457.9705505371094\n",
      "place 230\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 230, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  129 , action :  3 action_prob :  [0.05529442057013512, 0.1731540560722351, 0.04021081700921059, 0.5810551047325134, 0.06793973594903946, 0.038891591131687164, 0.043454360216856]\n",
      "extrinsic_reward :  2 intrinsic_reward :  442.0303955078125\n",
      "place 232\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 232, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  130 , action :  1 action_prob :  [0.05570684373378754, 0.17348460853099823, 0.04056059941649437, 0.5788268446922302, 0.0683559775352478, 0.03925587609410286, 0.04380929842591286]\n",
      "extrinsic_reward :  2 intrinsic_reward :  451.23211669921875\n",
      "place 234\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 234, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  131 , action :  1 action_prob :  [0.05511223152279854, 0.17301563918590546, 0.04005894064903259, 0.5820205807685852, 0.06776037812232971, 0.03873119875788689, 0.043300990015268326]\n",
      "extrinsic_reward :  2 intrinsic_reward :  452.0374755859375\n",
      "place 236\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 236, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  132 , action :  3 action_prob :  [0.05493855103850365, 0.1728610396385193, 0.039912786334753036, 0.5829748511314392, 0.06758437305688858, 0.03857608512043953, 0.04315227270126343]\n",
      "extrinsic_reward :  2 intrinsic_reward :  447.55987548828125\n",
      "place 238\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 238, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  133 , action :  3 action_prob :  [0.055563945323228836, 0.17334969341754913, 0.04044606164097786, 0.5795980095863342, 0.06822562962770462, 0.03911589831113815, 0.04370081052184105]\n",
      "extrinsic_reward :  2 intrinsic_reward :  447.03179931640625\n",
      "place 240\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 240, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  134 , action :  3 action_prob :  [0.05587944760918617, 0.17360343039035797, 0.04071149602532387, 0.5778999328613281, 0.0685393437743187, 0.039397936314344406, 0.0439683198928833]\n",
      "extrinsic_reward :  2 intrinsic_reward :  444.52996826171875\n",
      "place 242\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 242, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  135 , action :  0 action_prob :  [0.055269137024879456, 0.17312927544116974, 0.04019428789615631, 0.5811772346496582, 0.06792544573545456, 0.0388612225651741, 0.043443381786346436]\n",
      "extrinsic_reward :  2 intrinsic_reward :  449.45013427734375\n",
      "place 244\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 244, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  136 , action :  3 action_prob :  [0.05504670366644859, 0.17294149100780487, 0.04000554978847504, 0.582391619682312, 0.06769862771034241, 0.038665544241666794, 0.04325048625469208]\n",
      "extrinsic_reward :  3 intrinsic_reward :  448.5993957519531\n",
      "place 247\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 247, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  137 , action :  3 action_prob :  [0.0556916743516922, 0.1734456717967987, 0.04055480286478996, 0.5789110064506531, 0.06835809350013733, 0.0392240546643734, 0.04381467029452324]\n",
      "extrinsic_reward :  2 intrinsic_reward :  451.54937744140625\n",
      "place 249\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 249, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  138 , action :  1 action_prob :  [0.05573216825723648, 0.17349867522716522, 0.040589071810245514, 0.5786665678024292, 0.06840039789676666, 0.03926350921392441, 0.043849557638168335]\n",
      "extrinsic_reward :  3 intrinsic_reward :  443.05810546875\n",
      "place 252\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 294, 'world': 1, 'x_pos': 252, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  139 , action :  3 action_prob :  [0.05559791624546051, 0.17338648438453674, 0.040474697947502136, 0.5793995261192322, 0.06826359778642654, 0.03914521262049675, 0.043732523918151855]\n",
      "extrinsic_reward :  1 intrinsic_reward :  453.6348876953125\n",
      "place 254\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 254, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  140 , action :  3 action_prob :  [0.05566157400608063, 0.17342251539230347, 0.04052826762199402, 0.5790752172470093, 0.0683269128203392, 0.03919777274131775, 0.043787866830825806]\n",
      "extrinsic_reward :  3 intrinsic_reward :  434.03961181640625\n",
      "place 257\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 257, 'x_pos_screen': 112, 'y_pos': 79}\n",
      "global_step :  141 , action :  3 action_prob :  [0.05598071590065956, 0.17367440462112427, 0.04079693555831909, 0.577363133430481, 0.06864377856254578, 0.039482276886701584, 0.04405872896313667]\n",
      "extrinsic_reward :  3 intrinsic_reward :  446.35260009765625\n",
      "place 260\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 260, 'x_pos_screen': 112, 'y_pos': 78}\n",
      "global_step :  142 , action :  3 action_prob :  [0.05560649186372757, 0.17338816821575165, 0.04048142582178116, 0.5793612599372864, 0.06827137619256973, 0.039151862263679504, 0.043739382177591324]\n",
      "extrinsic_reward :  3 intrinsic_reward :  452.42822265625\n",
      "place 263\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 263, 'x_pos_screen': 112, 'y_pos': 77}\n",
      "global_step :  143 , action :  4 action_prob :  [0.05555156618356705, 0.1733406037092209, 0.040430158376693726, 0.579678475856781, 0.0682077631354332, 0.03910679742693901, 0.043684691190719604]\n",
      "extrinsic_reward :  3 intrinsic_reward :  452.027099609375\n",
      "place 266\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 266, 'x_pos_screen': 112, 'y_pos': 75}\n",
      "global_step :  144 , action :  3 action_prob :  [0.05566931515932083, 0.1734464317560196, 0.04053102061152458, 0.5790250301361084, 0.06832858920097351, 0.03921278193593025, 0.043786901980638504]\n",
      "extrinsic_reward :  3 intrinsic_reward :  449.14434814453125\n",
      "place 269\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 269, 'x_pos_screen': 112, 'y_pos': 73}\n",
      "global_step :  145 , action :  1 action_prob :  [0.055011406540870667, 0.17291338741779327, 0.03997013717889786, 0.5826002955436707, 0.06765374541282654, 0.038640428334474564, 0.043210532516241074]\n",
      "extrinsic_reward :  3 intrinsic_reward :  443.4607849121094\n",
      "place 272\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 272, 'x_pos_screen': 112, 'y_pos': 70}\n",
      "global_step :  146 , action :  4 action_prob :  [0.055325668305158615, 0.17317785322666168, 0.040232960134744644, 0.5808995962142944, 0.06796657294034958, 0.038921333849430084, 0.043476056307554245]\n",
      "extrinsic_reward :  3 intrinsic_reward :  446.7463073730469\n",
      "place 275\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 275, 'x_pos_screen': 112, 'y_pos': 67}\n",
      "global_step :  147 , action :  3 action_prob :  [0.055069029331207275, 0.17296230792999268, 0.04001561179757118, 0.5822979807853699, 0.06770598143339157, 0.03869495913386345, 0.04325416311621666]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "extrinsic_reward :  3 intrinsic_reward :  447.98126220703125\n",
      "place 278\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 278, 'x_pos_screen': 112, 'y_pos': 63}\n",
      "global_step :  148 , action :  3 action_prob :  [0.05510670691728592, 0.1730078160762787, 0.04004213958978653, 0.5821004509925842, 0.06772929430007935, 0.038741689175367355, 0.04327185079455376]\n",
      "extrinsic_reward :  3 intrinsic_reward :  431.1484375\n",
      "place 281\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 281, 'x_pos_screen': 112, 'y_pos': 59}\n",
      "global_step :  149 , action :  3 action_prob :  [0.05534571781754494, 0.1731952279806137, 0.04024358093738556, 0.5808191299438477, 0.06796853244304657, 0.03895222768187523, 0.04347562789916992]\n",
      "extrinsic_reward :  3 intrinsic_reward :  429.13250732421875\n",
      "place 284\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 284, 'x_pos_screen': 112, 'y_pos': 54}\n",
      "global_step :  150 , action :  3 action_prob :  [0.0549505352973938, 0.17287476360797882, 0.039912182837724686, 0.5829461812973022, 0.06757384538650513, 0.038602810353040695, 0.04313969612121582]\n",
      "extrinsic_reward :  3 intrinsic_reward :  442.1252746582031\n",
      "place 287\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 287, 'x_pos_screen': 112, 'y_pos': 49}\n",
      "global_step :  151 , action :  3 action_prob :  [0.05476725473999977, 0.1727319210767746, 0.039750564843416214, 0.5839613676071167, 0.06736873090267181, 0.03845644369721413, 0.04296371340751648]\n",
      "extrinsic_reward :  3 intrinsic_reward :  442.1550598144531\n",
      "place 290\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 290, 'x_pos_screen': 112, 'y_pos': 44}\n",
      "global_step :  152 , action :  4 action_prob :  [0.054911237210035324, 0.17284400761127472, 0.03987419232726097, 0.5831796526908875, 0.06752033531665802, 0.03857768699526787, 0.04309285804629326]\n",
      "extrinsic_reward :  3 intrinsic_reward :  437.40557861328125\n",
      "place 293\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 293, 'x_pos_screen': 112, 'y_pos': 39}\n",
      "global_step :  153 , action :  0 action_prob :  [0.05444241687655449, 0.17246611416339874, 0.03947249799966812, 0.5857357978820801, 0.06702949851751328, 0.03817860037088394, 0.042675185948610306]\n",
      "extrinsic_reward :  3 intrinsic_reward :  433.41302490234375\n",
      "place 296\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 296, 'x_pos_screen': 112, 'y_pos': 34}\n",
      "global_step :  154 , action :  3 action_prob :  [0.05464696139097214, 0.17264051735401154, 0.03964240103960037, 0.5846353769302368, 0.06722752004861832, 0.03836572915315628, 0.04284154251217842]\n",
      "extrinsic_reward :  3 intrinsic_reward :  439.12890625\n",
      "place 299\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 299, 'x_pos_screen': 112, 'y_pos': 29}\n",
      "global_step :  155 , action :  3 action_prob :  [0.054346777498722076, 0.17235969007015228, 0.03939386457204819, 0.5862833857536316, 0.06693203747272491, 0.03809047490358353, 0.04259377717971802]\n",
      "extrinsic_reward :  3 intrinsic_reward :  431.5587463378906\n",
      "place 302\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 302, 'x_pos_screen': 112, 'y_pos': 24}\n",
      "global_step :  156 , action :  1 action_prob :  [0.05465067923069, 0.17263652384281158, 0.039639320224523544, 0.584652304649353, 0.0672135055065155, 0.0383792519569397, 0.042828332632780075]\n",
      "extrinsic_reward :  3 intrinsic_reward :  416.27557373046875\n",
      "place 305\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 305, 'x_pos_screen': 112, 'y_pos': 19}\n",
      "global_step :  157 , action :  3 action_prob :  [0.05477878451347351, 0.17272590100765228, 0.039748553186655045, 0.5839751362800598, 0.06734375655651093, 0.03848809748888016, 0.04293964430689812]\n",
      "extrinsic_reward :  3 intrinsic_reward :  409.7021484375\n",
      "place 308\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 308, 'x_pos_screen': 112, 'y_pos': 14}\n",
      "global_step :  158 , action :  3 action_prob :  [0.054240819066762924, 0.17227661609649658, 0.039302758872509, 0.5868624448776245, 0.06681720167398453, 0.03800555318593979, 0.04249466210603714]\n",
      "extrinsic_reward :  3 intrinsic_reward :  416.45672607421875\n",
      "place 311\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 293, 'world': 1, 'x_pos': 311, 'x_pos_screen': 112, 'y_pos': 9}\n",
      "global_step :  159 , action :  6 action_prob :  [0.054683439433574677, 0.17266246676445007, 0.03966432064771652, 0.5844905376434326, 0.06723705679178238, 0.03841622173786163, 0.0428459532558918]\n",
      "extrinsic_reward :  2 intrinsic_reward :  410.118408203125\n",
      "place 314\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 292, 'world': 1, 'x_pos': 314, 'x_pos_screen': 112, 'y_pos': 4}\n",
      "epoch :  4 , global_step :  160\n",
      "global_step :  1 , action :  1 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.56402587890625\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  2 , action :  1 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.56402587890625\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  3 , action :  0 action_prob :  [0.05983036383986473, 0.17653553187847137, 0.0441148541867733, 0.5566235184669495, 0.0725478008389473, 0.042902544140815735, 0.04744540527462959]\n",
      "extrinsic_reward :  0 intrinsic_reward :  410.9014892578125\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  4 , action :  6 action_prob :  [0.05981843173503876, 0.17652776837348938, 0.0441044345498085, 0.5566869974136353, 0.07253562659025192, 0.04289211705327034, 0.047434646636247635]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.946533203125\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  5 , action :  4 action_prob :  [0.05979273095726967, 0.1765090972185135, 0.04408237338066101, 0.5568243265151978, 0.0725105032324791, 0.04286839812994003, 0.04741258919239044]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.946533203125\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 79}\n",
      "global_step :  6 , action :  2 action_prob :  [0.05979273095726967, 0.1765090972185135, 0.04408237338066101, 0.5568243265151978, 0.0725105032324791, 0.04286839812994003, 0.04741258919239044]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.9494934082031\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 84}\n",
      "global_step :  7 , action :  3 action_prob :  [0.05981937050819397, 0.17652717232704163, 0.04410558566451073, 0.5566821694374084, 0.07253731787204742, 0.0428922139108181, 0.047436222434043884]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.7156982421875\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 89}\n",
      "global_step :  8 , action :  3 action_prob :  [0.05974404886364937, 0.17647522687911987, 0.04403933137655258, 0.5570882558822632, 0.07245974242687225, 0.04282563179731369, 0.0473678894340992]\n",
      "extrinsic_reward :  0 intrinsic_reward :  413.0935974121094\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 93}\n",
      "global_step :  9 , action :  3 action_prob :  [0.05971038341522217, 0.17645451426506042, 0.04400930553674698, 0.557267963886261, 0.0724242553114891, 0.042797137051820755, 0.047336362302303314]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.6022644042969\n",
      "place 40\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 40, 'x_pos_screen': 40, 'y_pos': 97}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  10 , action :  6 action_prob :  [0.05961605906486511, 0.17639197409152985, 0.043925948441028595, 0.557774543762207, 0.07232643663883209, 0.04271500185132027, 0.04724998399615288]\n",
      "extrinsic_reward :  1 intrinsic_reward :  413.97406005859375\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 100}\n",
      "global_step :  11 , action :  4 action_prob :  [0.05967261642217636, 0.17643171548843384, 0.0439753383398056, 0.5574703812599182, 0.07238389551639557, 0.0427655465900898, 0.04730050638318062]\n",
      "extrinsic_reward :  0 intrinsic_reward :  412.70538330078125\n",
      "place 41\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 41, 'x_pos_screen': 41, 'y_pos': 103}\n",
      "global_step :  12 , action :  1 action_prob :  [0.059678398072719574, 0.17643551528453827, 0.0439801886677742, 0.5574402809143066, 0.07238941639661789, 0.04277077689766884, 0.04730541631579399]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.3945007324219\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 104}\n",
      "global_step :  13 , action :  3 action_prob :  [0.059669673442840576, 0.176429882645607, 0.04397227242588997, 0.5574878454208374, 0.07237987220287323, 0.04276350140571594, 0.047296978533267975]\n",
      "extrinsic_reward :  0 intrinsic_reward :  411.5827941894531\n",
      "place 42\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 42, 'x_pos_screen': 42, 'y_pos': 106}\n",
      "global_step :  14 , action :  3 action_prob :  [0.05969256907701492, 0.17644497752189636, 0.04399259015917778, 0.5573647618293762, 0.07240372151136398, 0.042783357203006744, 0.04731806740164757]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.85479736328125\n",
      "place 43\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 43, 'x_pos_screen': 43, 'y_pos': 106}\n",
      "global_step :  15 , action :  1 action_prob :  [0.059768348932266235, 0.1764957755804062, 0.0440593883395195, 0.5569576621055603, 0.0724819004535675, 0.04284980893135071, 0.04738706350326538]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.916259765625\n",
      "place 44\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 44, 'x_pos_screen': 44, 'y_pos': 106}\n",
      "global_step :  16 , action :  3 action_prob :  [0.05975944176316261, 0.17648901045322418, 0.04405168443918228, 0.5570061206817627, 0.07247290760278702, 0.04284169524908066, 0.047379203140735626]\n",
      "extrinsic_reward :  1 intrinsic_reward :  411.9704895019531\n",
      "place 45\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 45, 'x_pos_screen': 45, 'y_pos': 105}\n",
      "global_step :  17 , action :  3 action_prob :  [0.05976269394159317, 0.17649133503437042, 0.04405440762639046, 0.5569888949394226, 0.07247599214315414, 0.042844731360673904, 0.04738190397620201]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.65167236328125\n",
      "place 46\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 46, 'x_pos_screen': 46, 'y_pos': 104}\n",
      "global_step :  18 , action :  3 action_prob :  [0.059706348925828934, 0.1764528900384903, 0.04400510713458061, 0.5572910308837891, 0.07241857051849365, 0.04279472306370735, 0.047331325709819794]\n",
      "extrinsic_reward :  1 intrinsic_reward :  412.4834289550781\n",
      "place 47\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 300, 'world': 1, 'x_pos': 47, 'x_pos_screen': 47, 'y_pos': 101}\n",
      "global_step :  19 , action :  3 action_prob :  [0.05973493680357933, 0.17647269368171692, 0.0440300852060318, 0.5571374297142029, 0.07244767993688583, 0.04282016679644585, 0.04735694080591202]\n",
      "extrinsic_reward :  0 intrinsic_reward :  409.2146301269531\n",
      "place 48\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 48, 'x_pos_screen': 48, 'y_pos': 99}\n",
      "global_step :  20 , action :  3 action_prob :  [0.05973821505904198, 0.17647503316402435, 0.044032804667949677, 0.5571203231811523, 0.07245080918073654, 0.04282313957810402, 0.04735969379544258]\n",
      "extrinsic_reward :  2 intrinsic_reward :  410.178955078125\n",
      "place 50\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 50, 'x_pos_screen': 50, 'y_pos': 95}\n",
      "global_step :  21 , action :  3 action_prob :  [0.05980101227760315, 0.1765178143978119, 0.044087737798690796, 0.5567837953567505, 0.07251464575529099, 0.04287901148200035, 0.04741593450307846]\n",
      "extrinsic_reward :  1 intrinsic_reward :  410.45135498046875\n",
      "place 51\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 51, 'x_pos_screen': 51, 'y_pos': 91}\n",
      "global_step :  22 , action :  3 action_prob :  [0.05975745618343353, 0.1764889359474182, 0.044049497693777084, 0.5570168495178223, 0.07246993482112885, 0.042840905487537384, 0.047376397997140884]\n",
      "extrinsic_reward :  2 intrinsic_reward :  408.816650390625\n",
      "place 53\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 53, 'x_pos_screen': 53, 'y_pos': 86}\n",
      "global_step :  23 , action :  3 action_prob :  [0.059766579419374466, 0.17649465799331665, 0.04405766353011131, 0.5569678544998169, 0.07247968018054962, 0.04284854605793953, 0.04738505184650421]\n",
      "extrinsic_reward :  2 intrinsic_reward :  410.9498596191406\n",
      "place 55\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 55, 'x_pos_screen': 55, 'y_pos': 81}\n",
      "global_step :  24 , action :  1 action_prob :  [0.05979171767830849, 0.17651145160198212, 0.044080134481191635, 0.5568317174911499, 0.0725061371922493, 0.04287039116024971, 0.047408394515514374]\n",
      "extrinsic_reward :  1 intrinsic_reward :  408.294189453125\n",
      "place 56\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 56, 'x_pos_screen': 56, 'y_pos': 79}\n",
      "global_step :  25 , action :  3 action_prob :  [0.05984131619334221, 0.17654471099376678, 0.04412348195910454, 0.5565666556358337, 0.07255662977695465, 0.04291412979364395, 0.047453027218580246]\n",
      "extrinsic_reward :  2 intrinsic_reward :  410.8534240722656\n",
      "place 58\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 58, 'x_pos_screen': 58, 'y_pos': 79}\n",
      "global_step :  26 , action :  3 action_prob :  [0.05988006293773651, 0.17657139897346497, 0.044157568365335464, 0.5563580989837646, 0.07259632647037506, 0.04294859245419502, 0.047487977892160416]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.8377685546875\n",
      "place 60\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 60, 'x_pos_screen': 60, 'y_pos': 79}\n",
      "global_step :  27 , action :  2 action_prob :  [0.05987466126680374, 0.17656762897968292, 0.044152889400720596, 0.5563869476318359, 0.07259096950292587, 0.04294365271925926, 0.04748326167464256]\n",
      "extrinsic_reward :  2 intrinsic_reward :  411.9974365234375\n",
      "place 62\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 62, 'x_pos_screen': 62, 'y_pos': 79}\n",
      "global_step :  28 , action :  3 action_prob :  [0.0598679780960083, 0.17656318843364716, 0.04414711147546768, 0.5564223527908325, 0.07258430868387222, 0.04293770343065262, 0.0474773608148098]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.2843017578125\n",
      "place 64\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 64, 'x_pos_screen': 64, 'y_pos': 85}\n",
      "global_step :  29 , action :  5 action_prob :  [0.05982517451047897, 0.1765338033437729, 0.044109929352998734, 0.55665123462677, 0.07254103571176529, 0.042899616062641144, 0.047439221292734146]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.31964111328125\n",
      "place 66\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 66, 'x_pos_screen': 66, 'y_pos': 91}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  30 , action :  3 action_prob :  [0.05982242524623871, 0.17653192579746246, 0.04410692676901817, 0.5566681623458862, 0.07253710180521011, 0.042897697538137436, 0.04743576422333717]\n",
      "extrinsic_reward :  2 intrinsic_reward :  413.2981872558594\n",
      "place 68\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 68, 'x_pos_screen': 68, 'y_pos': 96}\n",
      "global_step :  31 , action :  3 action_prob :  [0.05980929359793663, 0.17652267217636108, 0.04409528523683548, 0.5567394495010376, 0.07252340763807297, 0.042886052280664444, 0.04742375388741493]\n",
      "extrinsic_reward :  3 intrinsic_reward :  414.3782958984375\n",
      "place 71\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 71, 'x_pos_screen': 71, 'y_pos': 100}\n",
      "global_step :  32 , action :  3 action_prob :  [0.05989186838269234, 0.1765792965888977, 0.04416729509830475, 0.5562973022460938, 0.07260718941688538, 0.0429595448076725, 0.047497574239969254]\n",
      "extrinsic_reward :  2 intrinsic_reward :  414.9434814453125\n",
      "place 73\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 73, 'x_pos_screen': 73, 'y_pos': 104}\n",
      "global_step :  33 , action :  1 action_prob :  [0.059860631823539734, 0.176558256149292, 0.0441400520503521, 0.5564641952514648, 0.07257536053657532, 0.042931992560625076, 0.047469500452280045]\n",
      "extrinsic_reward :  2 intrinsic_reward :  412.3881530761719\n",
      "place 75\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 75, 'x_pos_screen': 75, 'y_pos': 106}\n",
      "global_step :  34 , action :  3 action_prob :  [0.05993788689374924, 0.1766105741262436, 0.04420718550682068, 0.5560523271560669, 0.07265298068523407, 0.04300111532211304, 0.047537945210933685]\n",
      "extrinsic_reward :  3 intrinsic_reward :  411.0286560058594\n",
      "place 78\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 78, 'x_pos_screen': 78, 'y_pos': 108}\n",
      "global_step :  35 , action :  1 action_prob :  [0.05990978702902794, 0.17659102380275726, 0.04418322443962097, 0.5562011003494263, 0.07262559980154037, 0.04297542944550514, 0.047513868659734726]\n",
      "extrinsic_reward :  3 intrinsic_reward :  412.25689697265625\n",
      "place 81\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 81, 'x_pos_screen': 81, 'y_pos': 109}\n",
      "global_step :  36 , action :  3 action_prob :  [0.06001296266913414, 0.1766619086265564, 0.044272396713495255, 0.5556520223617554, 0.07272771745920181, 0.04306920990347862, 0.047603849321603775]\n",
      "extrinsic_reward :  3 intrinsic_reward :  411.96820068359375\n",
      "place 84\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 84, 'x_pos_screen': 82, 'y_pos': 109}\n",
      "global_step :  37 , action :  0 action_prob :  [0.05997159704566002, 0.17663314938545227, 0.04423714429140091, 0.555870771408081, 0.07268762588500977, 0.043031126260757446, 0.047568581998348236]\n",
      "extrinsic_reward :  3 intrinsic_reward :  397.13006591796875\n",
      "place 87\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 87, 'x_pos_screen': 83, 'y_pos': 108}\n",
      "global_step :  38 , action :  1 action_prob :  [0.05967909097671509, 0.17642569541931152, 0.04399741441011429, 0.5573863983154297, 0.07242401689291, 0.04274913668632507, 0.04733826220035553]\n",
      "extrinsic_reward :  2 intrinsic_reward :  406.3278503417969\n",
      "place 89\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 299, 'world': 1, 'x_pos': 89, 'x_pos_screen': 84, 'y_pos': 107}\n",
      "global_step :  39 , action :  3 action_prob :  [0.0590246319770813, 0.17596934735774994, 0.04342703893780708, 0.5609000325202942, 0.07175696641206741, 0.04217009246349335, 0.046751901507377625]\n",
      "extrinsic_reward :  2 intrinsic_reward :  426.0229187011719\n",
      "place 92\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 92, 'x_pos_screen': 85, 'y_pos': 104}\n",
      "global_step :  40 , action :  1 action_prob :  [0.06026202067732811, 0.17682337760925293, 0.0444989837706089, 0.5542955994606018, 0.07299746572971344, 0.04327880218625069, 0.04784383252263069]\n",
      "extrinsic_reward :  3 intrinsic_reward :  418.158203125\n",
      "place 95\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 95, 'x_pos_screen': 86, 'y_pos': 101}\n",
      "global_step :  41 , action :  3 action_prob :  [0.06069780886173248, 0.17710267007350922, 0.04487735405564308, 0.5519943237304688, 0.07342999428510666, 0.04367053881287575, 0.04822729900479317]\n",
      "extrinsic_reward :  3 intrinsic_reward :  398.61083984375\n",
      "place 98\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 98, 'x_pos_screen': 87, 'y_pos': 97}\n",
      "global_step :  42 , action :  3 action_prob :  [0.058530814945697784, 0.1756189614534378, 0.042997777462005615, 0.5635554194450378, 0.07125258445739746, 0.04173313081264496, 0.04631126672029495]\n",
      "extrinsic_reward :  3 intrinsic_reward :  421.23577880859375\n",
      "place 101\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 101, 'x_pos_screen': 88, 'y_pos': 92}\n",
      "global_step :  43 , action :  6 action_prob :  [0.05929792299866676, 0.17615929245948792, 0.043668683618307114, 0.5594213604927063, 0.0720425620675087, 0.0424075610935688, 0.04700266942381859]\n",
      "extrinsic_reward :  3 intrinsic_reward :  423.459228515625\n",
      "place 104\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 104, 'x_pos_screen': 89, 'y_pos': 87}\n",
      "global_step :  44 , action :  3 action_prob :  [0.05964672565460205, 0.17639797925949097, 0.04397333413362503, 0.5575546026229858, 0.07239632308483124, 0.04271651804447174, 0.047314517199993134]\n",
      "extrinsic_reward :  3 intrinsic_reward :  422.4017028808594\n",
      "place 107\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 107, 'x_pos_screen': 90, 'y_pos': 82}\n",
      "global_step :  45 , action :  6 action_prob :  [0.059997186064720154, 0.17663156986236572, 0.04427412152290344, 0.555705189704895, 0.07274053990840912, 0.043033212423324585, 0.0476180836558342]\n",
      "extrinsic_reward :  3 intrinsic_reward :  414.51715087890625\n",
      "place 110\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 110, 'x_pos_screen': 91, 'y_pos': 79}\n",
      "global_step :  46 , action :  2 action_prob :  [0.05786041170358658, 0.17513935267925262, 0.04242002218961716, 0.5671436190605164, 0.07057572901248932, 0.041139595210552216, 0.04572119191288948]\n",
      "extrinsic_reward :  3 intrinsic_reward :  436.2476501464844\n",
      "place 113\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 113, 'x_pos_screen': 92, 'y_pos': 79}\n",
      "global_step :  47 , action :  2 action_prob :  [0.058552756905555725, 0.17563779652118683, 0.043025463819503784, 0.5633987784385681, 0.07129482924938202, 0.04174244403839111, 0.046347927302122116]\n",
      "extrinsic_reward :  3 intrinsic_reward :  434.84014892578125\n",
      "place 116\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 116, 'x_pos_screen': 93, 'y_pos': 85}\n",
      "global_step :  48 , action :  3 action_prob :  [0.058730579912662506, 0.17576579749584198, 0.04317760467529297, 0.5624509453773499, 0.07147087901830673, 0.04190170392394066, 0.04650240018963814]\n",
      "extrinsic_reward :  3 intrinsic_reward :  428.173095703125\n",
      "place 119\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 119, 'x_pos_screen': 94, 'y_pos': 91}\n",
      "global_step :  49 , action :  3 action_prob :  [0.05901195481419563, 0.17596343159675598, 0.04341810569167137, 0.5609579682350159, 0.07174782454967499, 0.042156219482421875, 0.04674453288316727]\n",
      "extrinsic_reward :  3 intrinsic_reward :  411.0434875488281\n",
      "place 122\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 122, 'x_pos_screen': 95, 'y_pos': 97}\n",
      "global_step :  50 , action :  3 action_prob :  [0.05736049637198448, 0.17476648092269897, 0.041991204023361206, 0.5698304772377014, 0.0700717344880104, 0.040694624185562134, 0.04528499394655228]\n",
      "extrinsic_reward :  3 intrinsic_reward :  436.9667053222656\n",
      "place 125\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 125, 'x_pos_screen': 96, 'y_pos': 102}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "global_step :  51 , action :  1 action_prob :  [0.057804934680461884, 0.1750953048467636, 0.04238089174032211, 0.5674107074737549, 0.07053947448730469, 0.04107797518372536, 0.04569076746702194]\n",
      "extrinsic_reward :  3 intrinsic_reward :  434.695556640625\n",
      "place 128\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 128, 'x_pos_screen': 97, 'y_pos': 106}\n",
      "global_step :  52 , action :  1 action_prob :  [0.05807214230298996, 0.17528285086154938, 0.04261305555701256, 0.5659791827201843, 0.07081256061792374, 0.04131057485938072, 0.045929569751024246]\n",
      "extrinsic_reward :  3 intrinsic_reward :  428.8128662109375\n",
      "place 131\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 131, 'x_pos_screen': 98, 'y_pos': 109}\n",
      "global_step :  53 , action :  3 action_prob :  [0.058218106627464294, 0.1753910630941391, 0.04273540526628494, 0.5652058720588684, 0.070953369140625, 0.04144427925348282, 0.046051811426877975]\n",
      "extrinsic_reward :  3 intrinsic_reward :  422.46759033203125\n",
      "place 134\n",
      "info {'coins': 0, 'flag_get': False, 'life': 2, 'score': 0, 'stage': 3, 'status': 'small', 'time': 298, 'world': 1, 'x_pos': 134, 'x_pos_screen': 99, 'y_pos': 111}\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-35-60cd9084f4eb>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     18\u001b[0m                 \u001b[0maction_prob\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mppo\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 20\u001b[1;33m                 \u001b[0maction_prob\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mppo\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     21\u001b[0m             \u001b[0mm\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCategorical\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maction_prob\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m             \u001b[0maction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msample\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-18-a84c4e972846>\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, x, dim)\u001b[0m\n\u001b[0;32m     53\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mactor\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzero_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     54\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdim\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 55\u001b[1;33m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     56\u001b[0m         \u001b[0maction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mactor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     57\u001b[0m         \u001b[0maction_prob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mF\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maction\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdim\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdim\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\unity\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    488\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 489\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    490\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\unity\\lib\\site-packages\\torch\\nn\\modules\\container.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m     90\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 92\u001b[1;33m             \u001b[0minput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     93\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\unity\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    488\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 489\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    490\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\unity\\lib\\site-packages\\torch\\nn\\modules\\linear.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m     65\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mweak_script_method\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 67\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     68\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     69\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\unity\\lib\\site-packages\\torch\\nn\\functional.py\u001b[0m in \u001b[0;36mlinear\u001b[1;34m(input, weight, bias)\u001b[0m\n\u001b[0;32m   1350\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m2\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mbias\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1351\u001b[0m         \u001b[1;31m# fused op is marginally faster\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1352\u001b[1;33m         \u001b[0mret\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maddmm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjit\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_unwrap_optional\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1353\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1354\u001b[0m         \u001b[0moutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "T_horizon = 20\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    global_step = 0\n",
    "    state = env.reset()\n",
    "    state = np.array(state)/255\n",
    "            #state = np.transpose(state,(2,0,1))\n",
    "    state = np.moveaxis(state, -1, 0)\n",
    "    state = torch.tensor(state).float()\n",
    "    state = state.unsqueeze(0)\n",
    "    done = False\n",
    "    while not done :\n",
    "        for t in range(T_horizon):\n",
    "            env.render()\n",
    "            global_step +=1\n",
    "\n",
    "            if gpu:\n",
    "                action_prob, _ , _ = model.ppo.forward(state.cuda())\n",
    "            else:\n",
    "                action_prob, _ , _ = model.ppo.forward(state)\n",
    "            m = Categorical(action_prob)\n",
    "            action = m.sample().item()\n",
    "            next_state, extrinsic_reward, done, info = env.step(action)\n",
    "            next_state = np.array(next_state)/255\n",
    "            next_state = np.moveaxis(next_state,-1,0)\n",
    "            next_state = torch.tensor(next_state).float()\n",
    "            next_state = next_state.unsqueeze(0)\n",
    "            if gpu:\n",
    "                predictor,target = model.rnd.forward(next_state.cuda())\n",
    "            else:\n",
    "                predictor,target = model.rnd.forward(next_state)\n",
    "            intrinsic_reward = (predictor - target).pow(2).sum(1) / 2\n",
    "            if info['time'] == 0 :\n",
    "                done = True\n",
    "                reward = -10.\n",
    "            if done :\n",
    "                print('epoch : ',epoch, ', global_step : ',global_step)\n",
    "                break\n",
    "            print('global_step : ',global_step,', action : ', action,'action_prob : ',action_prob.tolist()[0])\n",
    "            print('extrinsic_reward : ',extrinsic_reward,'intrinsic_reward : ',intrinsic_reward.item())\n",
    "            print('place',info['x_pos'])\n",
    "            print('info',info)\n",
    "            state = next_state\n",
    "    #env.render()\n",
    "\n",
    "#env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "unity",
   "language": "python",
   "name": "unity"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}