keras_dqn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.keras as keras\n",
    "import numpy as np\n",
    "import gym\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.set_floatx('float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent(keras.Model):\n",
    "    \n",
    "    def __init__(self, h_s, n_action):\n",
    "        super(Agent, self).__init__()\n",
    "        self.fc1 = keras.layers.Dense(h_s, activation='relu', name='layer_1')\n",
    "        self.fc2 = keras.layers.Dense(n_action, activation=None, name='dqn_output')\n",
    "        \n",
    "    def call(self, inputs):\n",
    "        x = self.fc1(inputs)\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# env = gym.make('CartPole-v0')\n",
    "# env.reset()\n",
    "# for _ in range(1000):\n",
    "#     env.render()\n",
    "#     _, rew, done, _ = env.step(env.action_space.sample()) # take a random action\n",
    "#     if done:\n",
    "#         env.reset()\n",
    "# env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN:\n",
    "    \n",
    "    def __init__(self, n_features, n_action, memory_limit, epsilon, params_change_pointer, gamma, learning_rate):\n",
    "        \n",
    "        self.n_features = n_features\n",
    "        self.n_action = n_action\n",
    "        self.experience_limit = memory_limit\n",
    "        self.experiene_counter = 0\n",
    "        self.epsilon = epsilon\n",
    "        self.params_change_pointer = params_change_pointer\n",
    "        self.gamma = gamma\n",
    "        self.learning_rate = learning_rate\n",
    "        self.batch_size = 100\n",
    "        self.learning_counter = 0\n",
    "        self.memory = np.zeros([self.experience_limit, self.n_features*2+2])\n",
    "        \n",
    "        self.build_models()\n",
    "    \n",
    "    def build_models(self):\n",
    "        \n",
    "        self.primary_network = Agent(10, self.n_action)       \n",
    "        self.target_network = Agent(10, self.n_action)\n",
    "        self.optimizer = keras.optimizers.Adam(learning_rate=self.learning_rate)\n",
    "        \n",
    "    def fit(self):\n",
    "        \n",
    "        if self.experiene_counter < self.experience_limit:\n",
    "            indices = np.random.choice(self.experiene_counter, size=self.batch_size)\n",
    "        else:\n",
    "            indices = np.random.choice(self.experience_limit, size=self.batch_size)\n",
    "        batch = self.memory[indices, :]\n",
    "        q_p = self.primary_network(batch[:, :self.n_features])\n",
    "        q_t = self.target_network(batch[:, -self.n_features:])\n",
    "        \n",
    "        q_eval = q_p.numpy().copy()\n",
    "        batch_idx = np.arange(self.batch_size, dtype=np.int32)\n",
    "        actions = self.memory[indices, self.n_features].astype(int)\n",
    "        rewards = self.memory[indices, self.n_features+1]\n",
    "        q_eval[batch_idx, actions] = rewards + self.gamma * np.max(q_t, axis=1)\n",
    "        \n",
    "        with tf.GradientTape() as tape:\n",
    "            prediction = self.primary_network(batch[:, :self.n_features])\n",
    "            loss = keras.losses.MSE(y_true=q_eval, y_pred=prediction)\n",
    "        gradients = tape.gradient(loss, self.primary_network.trainable_variables)\n",
    "        self.optimizer.apply_gradients(zip(gradients, self.primary_network.trainable_variables))\n",
    "        \n",
    "        #self.primary_network.fit(batch[:, :self.n_features], q_eval, epochs=1, verbose=0)\n",
    "        \n",
    "        if self.epsilon > 0.1:\n",
    "            self.epsilon -= 0.0002\n",
    "        if self.learning_counter % self.params_change_pointer == 0:\n",
    "            self.target_params_change()\n",
    "        self.learning_counter += 1\n",
    "    \n",
    "    def epsilon_greedy(self, obs):\n",
    "        if np.random.uniform(low=0, high=1) > self.epsilon:\n",
    "            return np.argmax(self.primary_network(obs[np.newaxis, :]))\n",
    "        else:\n",
    "            return np.random.choice(self.n_action)\n",
    "    \n",
    "    def store_experience(self, obs, a, r, obs_):\n",
    "        index = self.experiene_counter % self.experience_limit\n",
    "        self.memory[index] = np.hstack((obs, [a, r], obs_))\n",
    "        self.experiene_counter += 1\n",
    "    \n",
    "    def target_params_change(self):\n",
    "        for i, layer in enumerate(self.primary_network.layers):\n",
    "            weights = layer.get_weights()\n",
    "            self.target_network.layers[i].set_weights(weights)\n",
    "        print('Target network parameters changed...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "dqn = DQN(env.observation_space.shape[0], env.action_space.n, 2000, 0.4, 100, 0.99, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPISODES = 150\n",
    "def train():\n",
    "    total_steps = 0\n",
    "    for episode in range(EPISODES):\n",
    "        obs = env.reset()\n",
    "        steps = 0\n",
    "        total_reward = 0\n",
    "        while True:\n",
    "            env.render()\n",
    "            a = dqn.epsilon_greedy(obs)\n",
    "            obs_, r, d, _ = env.step(a)\n",
    "            x, vel, angle, angle_vel = obs_\n",
    "            r1 = (env.x_threshold - abs(x))/env.x_threshold-0.8\n",
    "            r2 = (env.theta_threshold_radians - abs(angle))/env.theta_threshold_radians - 0.5\n",
    "            r = r1 + r2\n",
    "            total_reward += r\n",
    "            dqn.store_experience(obs, a, r, obs_)\n",
    "            if total_steps > 1000:\n",
    "                dqn.fit()\n",
    "            if d:\n",
    "                break\n",
    "            obs = obs_\n",
    "            steps += 1\n",
    "            total_steps += 1\n",
    "        print(\"Episode {} is completed at epsilon {} with reward {} in steps {}\".format(episode+1, dqn.epsilon,\n",
    "                                                                                        total_reward, steps))\n",
    "    env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 1 is completed at epsilon 0.4 with reward 2.703025730927006 in steps 12\n",
      "Episode 2 is completed at epsilon 0.4 with reward 2.3109763157241137 in steps 8\n",
      "Episode 3 is completed at epsilon 0.4 with reward 4.351363436519171 in steps 13\n",
      "Episode 4 is completed at epsilon 0.4 with reward 3.058969795487201 in steps 10\n",
      "Episode 5 is completed at epsilon 0.4 with reward 2.570171420438006 in steps 14\n",
      "Episode 6 is completed at epsilon 0.4 with reward 1.5258085135832538 in steps 7\n",
      "Episode 7 is completed at epsilon 0.4 with reward 2.117470652380274 in steps 10\n",
      "Episode 8 is completed at epsilon 0.4 with reward 1.913066454681036 in steps 9\n",
      "Episode 9 is completed at epsilon 0.4 with reward 2.4587688859524257 in steps 10\n",
      "Episode 10 is completed at epsilon 0.4 with reward 1.5031734405181285 in steps 11\n",
      "Episode 11 is completed at epsilon 0.4 with reward 1.5377572625075513 in steps 7\n",
      "Episode 12 is completed at epsilon 0.4 with reward 2.9201138611425295 in steps 10\n",
      "Episode 13 is completed at epsilon 0.4 with reward 4.400162075174534 in steps 12\n",
      "Episode 14 is completed at epsilon 0.4 with reward 1.4846625240579472 in steps 8\n",
      "Episode 15 is completed at epsilon 0.4 with reward 1.6526032936730686 in steps 11\n",
      "Episode 16 is completed at epsilon 0.4 with reward 2.6239701974841014 in steps 8\n",
      "Episode 17 is completed at epsilon 0.4 with reward 1.2426341432505636 in steps 9\n",
      "Episode 18 is completed at epsilon 0.4 with reward 1.5105157109122187 in steps 8\n",
      "Episode 19 is completed at epsilon 0.4 with reward 4.690864417372082 in steps 12\n",
      "Episode 20 is completed at epsilon 0.4 with reward 2.452413520467002 in steps 8\n",
      "Episode 21 is completed at epsilon 0.4 with reward 1.538751529996497 in steps 9\n",
      "Episode 22 is completed at epsilon 0.4 with reward 4.065613424215603 in steps 12\n",
      "Episode 23 is completed at epsilon 0.4 with reward 2.2931264427642133 in steps 10\n",
      "Episode 24 is completed at epsilon 0.4 with reward 4.197476560817027 in steps 15\n",
      "Episode 25 is completed at epsilon 0.4 with reward 3.098640293077514 in steps 11\n",
      "Episode 26 is completed at epsilon 0.4 with reward 2.7316130959470986 in steps 11\n",
      "Episode 27 is completed at epsilon 0.4 with reward 2.758019095061086 in steps 9\n",
      "Episode 28 is completed at epsilon 0.4 with reward 2.822464081325591 in steps 11\n",
      "Episode 29 is completed at epsilon 0.4 with reward 2.310213308656233 in steps 10\n",
      "Episode 30 is completed at epsilon 0.4 with reward 5.662532023992657 in steps 17\n",
      "Episode 31 is completed at epsilon 0.4 with reward 2.555868227087336 in steps 8\n",
      "Episode 32 is completed at epsilon 0.4 with reward 1.632152854224065 in steps 8\n",
      "Episode 33 is completed at epsilon 0.4 with reward 1.6343982975225695 in steps 8\n",
      "Episode 34 is completed at epsilon 0.4 with reward 2.5943563492158477 in steps 8\n",
      "Episode 35 is completed at epsilon 0.4 with reward 2.7246164446830843 in steps 12\n",
      "Episode 36 is completed at epsilon 0.4 with reward 1.0806680914916915 in steps 8\n",
      "Episode 37 is completed at epsilon 0.4 with reward 1.505635122204208 in steps 9\n",
      "Episode 38 is completed at epsilon 0.4 with reward 1.1872518943522739 in steps 7\n",
      "Episode 39 is completed at epsilon 0.4 with reward 2.3758772179688554 in steps 15\n",
      "Episode 40 is completed at epsilon 0.4 with reward 3.906973654858285 in steps 11\n",
      "Episode 41 is completed at epsilon 0.4 with reward 1.4530938989576594 in steps 10\n",
      "Episode 42 is completed at epsilon 0.4 with reward 2.997225922511867 in steps 12\n",
      "Episode 43 is completed at epsilon 0.4 with reward 3.1053599037800885 in steps 11\n",
      "Episode 44 is completed at epsilon 0.4 with reward 2.8486953468012324 in steps 10\n",
      "Episode 45 is completed at epsilon 0.4 with reward 2.3937163334193032 in steps 12\n",
      "Episode 46 is completed at epsilon 0.4 with reward 1.0400057753687069 in steps 11\n",
      "Episode 47 is completed at epsilon 0.4 with reward 1.3911255501682644 in steps 8\n",
      "Episode 48 is completed at epsilon 0.4 with reward 1.7975278118723588 in steps 8\n",
      "Episode 49 is completed at epsilon 0.4 with reward 2.6345659357608353 in steps 10\n",
      "Episode 50 is completed at epsilon 0.4 with reward 1.5717302889577933 in steps 8\n",
      "Episode 51 is completed at epsilon 0.4 with reward 2.3100003618367486 in steps 9\n",
      "Episode 52 is completed at epsilon 0.4 with reward 3.179557946498921 in steps 11\n",
      "Episode 53 is completed at epsilon 0.4 with reward 3.6203200344041764 in steps 10\n",
      "Episode 54 is completed at epsilon 0.4 with reward 1.3696560065826948 in steps 7\n",
      "Episode 55 is completed at epsilon 0.4 with reward 2.1275521385988956 in steps 10\n",
      "Episode 56 is completed at epsilon 0.4 with reward 3.561019121384587 in steps 11\n",
      "Episode 57 is completed at epsilon 0.4 with reward 2.383742555749131 in steps 10\n",
      "Episode 58 is completed at epsilon 0.4 with reward 2.5094240239217123 in steps 10\n",
      "Episode 59 is completed at epsilon 0.4 with reward 2.7899496444633693 in steps 10\n",
      "Episode 60 is completed at epsilon 0.4 with reward 1.945131344182124 in steps 9\n",
      "Episode 61 is completed at epsilon 0.4 with reward 1.872904126193256 in steps 8\n",
      "Episode 62 is completed at epsilon 0.4 with reward 3.4637523717851635 in steps 12\n",
      "Episode 63 is completed at epsilon 0.4 with reward 1.2390307348432343 in steps 7\n",
      "Episode 64 is completed at epsilon 0.4 with reward 2.3305321582172653 in steps 9\n",
      "Episode 65 is completed at epsilon 0.4 with reward 0.9760280017005609 in steps 8\n",
      "Episode 66 is completed at epsilon 0.4 with reward 7.9869095618124435 in steps 18\n",
      "Episode 67 is completed at epsilon 0.4 with reward 2.72167366558051 in steps 10\n",
      "Episode 68 is completed at epsilon 0.4 with reward 2.708841342250228 in steps 12\n",
      "Episode 69 is completed at epsilon 0.4 with reward 2.703869894189771 in steps 9\n",
      "Episode 70 is completed at epsilon 0.4 with reward 3.7388117419141653 in steps 11\n",
      "Episode 71 is completed at epsilon 0.4 with reward 1.6300209832572352 in steps 9\n",
      "Episode 72 is completed at epsilon 0.4 with reward 1.0320786306663297 in steps 10\n",
      "Episode 73 is completed at epsilon 0.4 with reward 3.7457878331332095 in steps 15\n",
      "Episode 74 is completed at epsilon 0.4 with reward 1.8737778252953603 in steps 9\n",
      "Episode 75 is completed at epsilon 0.4 with reward 2.4919850668598746 in steps 8\n",
      "Episode 76 is completed at epsilon 0.4 with reward 3.0461101766241665 in steps 11\n",
      "Episode 77 is completed at epsilon 0.4 with reward 8.520361827479016 in steps 18\n",
      "Episode 78 is completed at epsilon 0.4 with reward 2.437023225273266 in steps 10\n",
      "Episode 79 is completed at epsilon 0.4 with reward 2.893100004506299 in steps 10\n",
      "Episode 80 is completed at epsilon 0.4 with reward 5.679059504971103 in steps 13\n",
      "Episode 81 is completed at epsilon 0.4 with reward 2.6618991412363684 in steps 10\n",
      "Episode 82 is completed at epsilon 0.4 with reward 2.451145906587631 in steps 13\n",
      "Episode 83 is completed at epsilon 0.4 with reward 4.877037768372474 in steps 13\n",
      "Episode 84 is completed at epsilon 0.4 with reward 2.1228493412866323 in steps 8\n",
      "Episode 85 is completed at epsilon 0.4 with reward 3.447543330402177 in steps 11\n",
      "Episode 86 is completed at epsilon 0.4 with reward 1.9473188520709976 in steps 8\n",
      "Episode 87 is completed at epsilon 0.4 with reward 0.9095241049838675 in steps 7\n",
      "Episode 88 is completed at epsilon 0.4 with reward 4.1835495945052505 in steps 11\n",
      "Episode 89 is completed at epsilon 0.4 with reward 3.1378917478691193 in steps 11\n",
      "Episode 90 is completed at epsilon 0.4 with reward 2.019241301071938 in steps 8\n",
      "Episode 91 is completed at epsilon 0.4 with reward 1.872920398227542 in steps 8\n",
      "Episode 92 is completed at epsilon 0.4 with reward 3.992487360554766 in steps 13\n",
      "Episode 93 is completed at epsilon 0.4 with reward 4.490198934381011 in steps 12\n",
      "Episode 94 is completed at epsilon 0.4 with reward 2.485542338774046 in steps 11\n",
      "Episode 95 is completed at epsilon 0.4 with reward 0.8304847737776165 in steps 8\n",
      "Episode 96 is completed at epsilon 0.4 with reward 2.813627546325471 in steps 11\n",
      "Episode 97 is completed at epsilon 0.4 with reward 1.220629470077697 in steps 7\n",
      "Episode 98 is completed at epsilon 0.4 with reward 2.3756258872017924 in steps 8\n",
      "Target network parameters changed...\n",
      "Episode 99 is completed at epsilon 0.3986000000000002 with reward 2.758804902020789 in steps 9\n",
      "Episode 100 is completed at epsilon 0.391000000000001 with reward 6.192920740318202 in steps 37\n",
      "Episode 101 is completed at epsilon 0.38900000000000123 with reward 3.07393303839562 in steps 9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 102 is completed at epsilon 0.38620000000000154 with reward 3.074088769257816 in steps 13\n",
      "Episode 103 is completed at epsilon 0.38360000000000183 with reward 2.831164439590162 in steps 12\n",
      "Episode 104 is completed at epsilon 0.38080000000000214 with reward 4.399405031715874 in steps 13\n",
      "Target network parameters changed...\n",
      "Episode 105 is completed at epsilon 0.37880000000000236 with reward 2.857107035432395 in steps 9\n",
      "Episode 106 is completed at epsilon 0.37700000000000256 with reward 1.0086793836750083 in steps 8\n",
      "Episode 107 is completed at epsilon 0.3746000000000028 with reward 1.3625694650415354 in steps 11\n",
      "Episode 108 is completed at epsilon 0.37240000000000306 with reward 2.9329829219942574 in steps 10\n",
      "Episode 109 is completed at epsilon 0.3702000000000033 with reward 2.3285375723242363 in steps 10\n",
      "Episode 110 is completed at epsilon 0.3684000000000035 with reward 1.491958071534467 in steps 8\n",
      "Episode 111 is completed at epsilon 0.3664000000000037 with reward 1.5224465485061098 in steps 9\n",
      "Episode 112 is completed at epsilon 0.364000000000004 with reward 3.5520114550861046 in steps 11\n",
      "Episode 113 is completed at epsilon 0.3622000000000042 with reward 2.1691104100287375 in steps 8\n",
      "Episode 114 is completed at epsilon 0.3600000000000044 with reward 3.1454879323466938 in steps 10\n",
      "Target network parameters changed...\n",
      "Episode 115 is completed at epsilon 0.35700000000000476 with reward 2.8191697615137334 in steps 14\n",
      "Episode 116 is completed at epsilon 0.35420000000000507 with reward 5.483344458004404 in steps 13\n",
      "Episode 117 is completed at epsilon 0.3484000000000057 with reward 3.233220842675139 in steps 28\n",
      "Episode 118 is completed at epsilon 0.34620000000000595 with reward 2.629107127037442 in steps 10\n",
      "Episode 119 is completed at epsilon 0.34000000000000663 with reward 3.8778000860742203 in steps 30\n",
      "Target network parameters changed...\n",
      "Episode 120 is completed at epsilon 0.3378000000000069 with reward 1.660552175021331 in steps 10\n",
      "Episode 121 is completed at epsilon 0.3320000000000075 with reward 8.77394970129171 in steps 28\n",
      "Episode 122 is completed at epsilon 0.32640000000000813 with reward 3.5127824524610896 in steps 27\n",
      "Target network parameters changed...\n",
      "Episode 123 is completed at epsilon 0.31240000000000967 with reward 6.765800246363988 in steps 69\n",
      "Episode 124 is completed at epsilon 0.30000000000001104 with reward 3.8119663564019337 in steps 61\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 125 is completed at epsilon 0.2706000000000143 with reward 44.66313384629453 in steps 146\n",
      "Target network parameters changed...\n",
      "Episode 126 is completed at epsilon 0.2560000000000159 with reward 31.79122435843452 in steps 72\n",
      "Target network parameters changed...\n",
      "Episode 127 is completed at epsilon 0.2206000000000157 with reward 55.49475679258342 in steps 176\n",
      "Target network parameters changed...\n",
      "Episode 128 is completed at epsilon 0.20380000000001522 with reward 9.121140455158843 in steps 83\n",
      "Target network parameters changed...\n",
      "Episode 129 is completed at epsilon 0.18400000000001465 with reward 17.560268690625183 in steps 98\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 130 is completed at epsilon 0.15720000000001388 with reward 42.36322794446613 in steps 133\n",
      "Target network parameters changed...\n",
      "Episode 131 is completed at epsilon 0.13940000000001337 with reward 10.259681972158095 in steps 88\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 132 is completed at epsilon 0.09980000000001224 with reward 95.55963445247622 in steps 199\n",
      "Target network parameters changed...\n",
      "Episode 133 is completed at epsilon 0.09980000000001224 with reward 26.251163066318284 in steps 101\n",
      "Target network parameters changed...\n",
      "Episode 134 is completed at epsilon 0.09980000000001224 with reward 29.68660591999072 in steps 101\n",
      "Target network parameters changed...\n",
      "Episode 135 is completed at epsilon 0.09980000000001224 with reward 56.748912303118445 in steps 145\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 136 is completed at epsilon 0.09980000000001224 with reward 78.98481058676416 in steps 191\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 137 is completed at epsilon 0.09980000000001224 with reward 110.06636283305224 in steps 199\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 138 is completed at epsilon 0.09980000000001224 with reward 111.67338817871254 in steps 199\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 139 is completed at epsilon 0.09980000000001224 with reward 90.48440127555017 in steps 199\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 140 is completed at epsilon 0.09980000000001224 with reward 128.94464067158282 in steps 199\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 141 is completed at epsilon 0.09980000000001224 with reward 118.20553097545154 in steps 199\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 142 is completed at epsilon 0.09980000000001224 with reward 121.82159460548053 in steps 199\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 143 is completed at epsilon 0.09980000000001224 with reward 132.5838780134058 in steps 199\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 144 is completed at epsilon 0.09980000000001224 with reward 101.86861517521615 in steps 199\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 145 is completed at epsilon 0.09980000000001224 with reward 120.74492761628387 in steps 199\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 146 is completed at epsilon 0.09980000000001224 with reward 113.5081382895698 in steps 199\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 147 is completed at epsilon 0.09980000000001224 with reward 125.63581468983378 in steps 199\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 148 is completed at epsilon 0.09980000000001224 with reward 108.05980701762606 in steps 199\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 149 is completed at epsilon 0.09980000000001224 with reward 111.35092046320256 in steps 199\n",
      "Target network parameters changed...\n",
      "Target network parameters changed...\n",
      "Episode 150 is completed at epsilon 0.09980000000001224 with reward 90.43206253975603 in steps 199\n"
     ]
    }
   ],
   "source": [
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}