trump_tweet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.enable_eager_execution()\n",
    "\n",
    "import numpy as np\n",
    "import os\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of text: 850909 characters\n"
     ]
    }
   ],
   "source": [
    "text = open('tweets.txt', 'rb').read().decode(encoding='utf-8')\n",
    "print ('Length of text: {} characters'.format(len(text)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106 unique characters\n"
     ]
    }
   ],
   "source": [
    "vocab = sorted(set(text))\n",
    "print ('{} unique characters'.format(len(vocab)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "char2idx = {u:i for i, u in enumerate(vocab)}\n",
    "idx2char = np.array(vocab)\n",
    "\n",
    "text_as_int = np.array([char2idx[c] for c in text])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  u'\\u064f': 101,\n",
      "  u' ':   1,\n",
      "  u'$':   5,\n",
      "  u'(':   8,\n",
      "  u',':  12,\n",
      "  u'0':  16,\n",
      "  u'4':  20,\n",
      "  u'8':  24,\n",
      "  u'@':  30,\n",
      "  u'D':  34,\n",
      "  u'\\u03c9':  92,\n",
      "  u'H':  38,\n",
      "  u'L':  42,\n",
      "  u'P':  46,\n",
      "  u'T':  50,\n",
      "  u'X':  54,\n",
      "  u'd':  63,\n",
      "  u'h':  67,\n",
      "  u'l':  71,\n",
      "  u'p':  75,\n",
      "  ...\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print('{')\n",
    "for char,_ in zip(char2idx, range(20)):\n",
    "    print('  {:4s}: {:3d},'.format(repr(char), char2idx[char]))\n",
    "print('  ...\\n}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T\n",
      "o\n",
      "d\n",
      "a\n",
      "y\n"
     ]
    }
   ],
   "source": [
    "seq_length = 100\n",
    "examples_per_epoch = len(text)//(seq_length+1)\n",
    "\n",
    "char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)\n",
    "\n",
    "for i in char_dataset.take(5):\n",
    "  print(idx2char[i.numpy()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "u'Today we express our deepest gratitude to all those who have served in our armed forces. #ThankAVet h'\n",
      "u'ttps://t.co/wPk7QWpK8Z\\nBusy day planned in New York. Will soon be making some very important decision'\n",
      "u's on the people who will be running our government!\\nLove the fact that the small groups of protesters'\n",
      "u' last night have passion for our great country. We will all come together and be proud!\\nJust had a ve'\n",
      "u'ry open and successful presidential election. Now professional protesters, incited by the media, are '\n"
     ]
    }
   ],
   "source": [
    "sequences = char_dataset.batch(seq_length+1, drop_remainder=True)\n",
    "\n",
    "for item in sequences.take(5):\n",
    "  print(repr(''.join(idx2char[item.numpy()])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_input_target(chunk):\n",
    "    input_text = chunk[:-1]\n",
    "    target_text = chunk[1:]\n",
    "    return input_text, target_text\n",
    "\n",
    "dataset = sequences.map(split_input_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Input data: ', \"u'Today we express our deepest gratitude to all those who have served in our armed forces. #ThankAVet '\")\n",
      "('Target data:', \"u'oday we express our deepest gratitude to all those who have served in our armed forces. #ThankAVet h'\")\n"
     ]
    }
   ],
   "source": [
    "for input_example, target_example in  dataset.take(1):\n",
    "  print ('Input data: ', repr(''.join(idx2char[input_example.numpy()])))\n",
    "  print ('Target data:', repr(''.join(idx2char[target_example.numpy()])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step    0\n",
      "  input: 50 (u'T')\n",
      "  expected output: 74 (u'o')\n",
      "Step    1\n",
      "  input: 74 (u'o')\n",
      "  expected output: 63 (u'd')\n",
      "Step    2\n",
      "  input: 63 (u'd')\n",
      "  expected output: 60 (u'a')\n",
      "Step    3\n",
      "  input: 60 (u'a')\n",
      "  expected output: 84 (u'y')\n",
      "Step    4\n",
      "  input: 84 (u'y')\n",
      "  expected output: 1 (u' ')\n",
      "<DatasetV1Adapter shapes: ((64, 100), (64, 100)), types: (tf.int64, tf.int64)>\n",
      "Vocab size = 106\n"
     ]
    }
   ],
   "source": [
    "for i, (input_idx, target_idx) in enumerate(zip(input_example[:5], target_example[:5])):\n",
    "    print(\"Step {:4d}\".format(i))\n",
    "    print(\"  input: {} ({:s})\".format(input_idx, repr(idx2char[input_idx])))\n",
    "    print(\"  expected output: {} ({:s})\".format(target_idx, repr(idx2char[target_idx])))\n",
    "\n",
    "BATCH_SIZE = 64\n",
    "\n",
    "BUFFER_SIZE = 10000\n",
    "\n",
    "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n",
    "\n",
    "print dataset\n",
    "\n",
    "vocab_size = len(vocab)\n",
    "print(\"Vocab size = \" + str(vocab_size))\n",
    "\n",
    "embedding_dim = 256\n",
    "\n",
    "rnn_units = 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(vocab_size, embedding_dim, rnn_units, batch_size):\n",
    "    model = tf.keras.Sequential([\n",
    "        tf.keras.layers.Embedding(vocab_size, embedding_dim,\n",
    "                                  batch_input_shape=[batch_size, None]),\n",
    "        tf.keras.layers.GRU(rnn_units,\n",
    "                            return_sequences=True,\n",
    "                            stateful=True,\n",
    "                            recurrent_initializer='glorot_uniform'),\n",
    "        tf.keras.layers.Dense(vocab_size)\n",
    "    ])\n",
    "    return model\n",
    "\n",
    "model = build_model(\n",
    "  vocab_size = len(vocab),\n",
    "  embedding_dim=embedding_dim,\n",
    "  rnn_units=rnn_units,\n",
    "  batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(TensorShape([Dimension(64), Dimension(100), Dimension(106)]), '# (batch_size, sequence_length, vocab_size)')\n",
      "[ 90  29   6  17  17  83  83 105 102  50  11  88  37  43 105  73   9  50\n",
      "  12  23  36  36  98  80  46  74  70  93 105  25 101  23  60  94  68  91\n",
      "  63  38  10  39  28   6  68  20  58  70  71  43  77  69  52  11  66  92\n",
      " 100  21  33  18  18   3  70  80  98  42  76  52  47  69  65  45  58  19\n",
      "  41  84   3  42  30  86  42  70  94  25  54  11  70 104   1  35  51   1\n",
      "  39  26  92  70  57  97  60  19  31   8]\n",
      "('Input: \\n', 'u\\' wouldnt be watching at all!!! Honestly!\"\\\\n\"@antSTACKSgrieco: @realDonaldTrump you were great in it!!\\'')\n",
      "()\n",
      "('Next Char Predictions: \\n', 'u\\'\\\\u0289?%11xx\\\\u06ea\\\\u066aT+}GM\\\\u06ean)T,7FF\\\\u0565uPok\\\\u044f\\\\u06ea9\\\\u064f7a\\\\u04d5i\\\\u0331dH*I=%i4]klMrjV+g\\\\u03c9\\\\u057b5C22\"ku\\\\u0565LqVQjfO]3Ky\"L@{Lk\\\\u04d59X+k\\\\u06e2 EU I:\\\\u03c9k[\\\\u0520a3A(\\'')\n",
      "('Prediction shape: ', TensorShape([Dimension(64), Dimension(100), Dimension(106)]), ' # (batch_size, sequence_length, vocab_size)')\n",
      "('scalar_loss:      ', 4.663801)\n"
     ]
    }
   ],
   "source": [
    "for input_example_batch, target_example_batch in dataset.take(1):\n",
    "  example_batch_predictions = model(input_example_batch)\n",
    "  print(example_batch_predictions.shape, \"# (batch_size, sequence_length, vocab_size)\")\n",
    "\n",
    "\n",
    "sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)\n",
    "sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()\n",
    "\n",
    "print(sampled_indices)\n",
    "\n",
    "print(\"Input: \\n\", repr(\"\".join(idx2char[input_example_batch[0]])))\n",
    "print()\n",
    "print(\"Next Char Predictions: \\n\", repr(\"\".join(idx2char[sampled_indices ])))\n",
    "\n",
    "def loss(labels, logits):\n",
    "  return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)\n",
    "\n",
    "example_batch_loss  = loss(target_example_batch, example_batch_predictions)\n",
    "print(\"Prediction shape: \", example_batch_predictions.shape, \" # (batch_size, sequence_length, vocab_size)\")\n",
    "print(\"scalar_loss:      \", example_batch_loss.numpy().mean())\n",
    "\n",
    "model.compile(optimizer='adam', loss=loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "WARNING:tensorflow:From /home/leo/.local/lib/python2.7/site-packages/tensorflow/python/ops/math_grad.py:1250: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "131/131 [==============================] - 421s 3s/step - loss: 3.1906\n",
      "Epoch 2/10\n",
      "131/131 [==============================] - 415s 3s/step - loss: 2.2689\n",
      "Epoch 3/10\n",
      "131/131 [==============================] - 431s 3s/step - loss: 1.9356\n",
      "Epoch 4/10\n",
      "131/131 [==============================] - 431s 3s/step - loss: 1.7324\n",
      "Epoch 5/10\n",
      "131/131 [==============================] - 449s 3s/step - loss: 1.5973\n",
      "Epoch 6/10\n",
      "131/131 [==============================] - 468s 4s/step - loss: 1.4983\n",
      "Epoch 7/10\n",
      "131/131 [==============================] - 468s 4s/step - loss: 1.4184\n",
      "Epoch 8/10\n",
      "131/131 [==============================] - 468s 4s/step - loss: 1.3482\n",
      "Epoch 9/10\n",
      "131/131 [==============================] - 470s 4s/step - loss: 1.2827\n",
      "Epoch 10/10\n",
      "131/131 [==============================] - 415s 3s/step - loss: 1.2190\n"
     ]
    }
   ],
   "source": [
    "checkpoint_dir = './training_checkpoints'\n",
    "# Name of the checkpoint files\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")\n",
    "\n",
    "checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(\n",
    "    filepath=checkpoint_prefix,\n",
    "    save_weights_only=True)\n",
    "\n",
    "EPOCHS = 10\n",
    "\n",
    "history = model.fit(dataset, epochs=EPOCHS, callbacks=[checkpoint_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_dir = './training_checkpoints'\n",
    "tf.train.latest_checkpoint(checkpoint_dir)\n",
    "\n",
    "\n",
    "model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)\n",
    "\n",
    "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n",
    "\n",
    "model.build(tf.TensorShape([1, None]))\n",
    "\n",
    "def generate_text(model, start_string):\n",
    "  \n",
    "  num_generate = 280\n",
    "\n",
    "  input_eval = [char2idx[s] for s in start_string]\n",
    "  input_eval = tf.expand_dims(input_eval, 0)\n",
    "\n",
    "  text_generated = []\n",
    "\n",
    "  temperature = 1.0\n",
    "\n",
    "  model.reset_states()\n",
    "  for i in range(num_generate):\n",
    "      predictions = model(input_eval)\n",
    "      \n",
    "      predictions = tf.squeeze(predictions, 0)\n",
    "\n",
    "      predictions = predictions / temperature\n",
    "      predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()\n",
    "\n",
    "      input_eval = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "      text_generated.append(idx2char[predicted_id])\n",
    "\n",
    "  return (start_string + ''.join(text_generated))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I WONT GOT #KetertyFlion\n",
      ".@Fiverryan, If (Faxter/EZSJFWAN to will come to the U Saide Hay agree. So Crooked Jonans), no action). https://t.co/oWVxLQUNMo\n",
      "John Kasich,  MA 7! 2 2513.22 https://t.co/X39HTfns48\n",
      "Going to MAKE AMERICA GREAT AGAIN!\n",
      "Frand lase night about by South Carolina\n"
     ]
    }
   ],
   "source": [
    "print(generate_text(model, start_string=u\"I \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('shakespeare.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}