RF_Basic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # CSV file I/O (e.g. pd.read_csv)\n",
    "from datetime import datetime\n",
    "import calendar\n",
    "import warnings\n",
    "from math import pi, sin, cos, sqrt, atan2, radians,asin\n",
    "import calendar\n",
    "import os # reading the input files we have access to\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import RandomForestRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of Training Data (1000000, 8)\n",
      "Shape of Testing Data (9914, 7)\n"
     ]
    }
   ],
   "source": [
    "train=pd.read_csv(\"input/train.csv\",nrows=1000000) #Has to be 10^6 for good performance [TODO]\n",
    "print(\"Shape of Training Data\",train.shape)\n",
    "test=pd.read_csv(\"input/test.csv\")\n",
    "print(\"Shape of Testing Data\", test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encodeDays(day_of_week):\n",
    "    day_dict={'Sunday':0,'Monday':1,'Tuesday':2,'Wednesday':3,'Thursday':4,'Friday':5,'Saturday':6}\n",
    "    return day_dict[day_of_week]\n",
    "def clean_data(data):\n",
    "    boundary={'min_lng':-74.263242,\n",
    "              'min_lat':40.573143,\n",
    "              'max_lng':-72.986532, \n",
    "              'max_lat':41.709555}\n",
    "    \n",
    "    data['pickup_datetime']=pd.to_datetime(data['pickup_datetime'],format='%Y-%m-%d %H:%M:%S UTC')\n",
    "    data['pickup_day']=data['pickup_datetime'].apply(lambda x:x.day)\n",
    "    #data['pickup_sin_hm'] = data['pickup_datetime'].apply(lambda x:sin(2*pi*(x.hour * 60 + x.minute)/1440))\n",
    "    data['pickup_hour']=data['pickup_datetime'].apply(lambda x:x.hour)\n",
    "    data['pickup_month']=data['pickup_datetime'].apply(lambda x:x.month)\n",
    "    data['pickup_year']=data['pickup_datetime'].apply(lambda x:x.year)\n",
    "    if 'fare_amount' in data.columns:\n",
    "        data=data[data['fare_amount']>=0]\n",
    "        data.loc[~((data.pickup_longitude >= boundary['min_lng'] ) & (data.pickup_longitude <= boundary['max_lng']) &\n",
    "            (data.pickup_latitude >= boundary['min_lat']) & (data.pickup_latitude <= boundary['max_lat']) &\n",
    "            (data.dropoff_longitude >= boundary['min_lng']) & (data.dropoff_longitude <= boundary['max_lng']) &\n",
    "            (data.dropoff_latitude >=boundary['min_lat']) & (data.dropoff_latitude <= boundary['max_lat'])),'is_outlier_loc']=1\n",
    "        data.loc[((data.pickup_longitude >= boundary['min_lng'] ) & (data.pickup_longitude <= boundary['max_lng']) &\n",
    "            (data.pickup_latitude >= boundary['min_lat']) & (data.pickup_latitude <= boundary['max_lat']) &\n",
    "            (data.dropoff_longitude >= boundary['min_lng']) & (data.dropoff_longitude <= boundary['max_lng']) &\n",
    "            (data.dropoff_latitude >=boundary['min_lat']) & (data.dropoff_latitude <= boundary['max_lat'])),'is_outlier_loc']=0\n",
    "\n",
    "    #print(\"Outlier vs Non Outlier Counts\")\n",
    "    #print(data['is_outlier_loc'].value_counts())\n",
    "\n",
    "    # Let us drop rows, where location is outlier\n",
    "        data=data.loc[data['is_outlier_loc']==0]\n",
    "        data.drop(['is_outlier_loc'],axis=1,inplace=True)\n",
    "    \n",
    "    data=data[data['passenger_count']<=8]\n",
    "    \n",
    "    weekday_names = {0: 'Monday',\n",
    "                 1: 'Tuesday',\n",
    "                 2: 'Wednesday',\n",
    "                 3: 'Thursday',\n",
    "                 4: 'Friday',\n",
    "                 5: 'Saturday',\n",
    "                 6: 'Sunday'}\n",
    "    #data = data.join(pd.get_dummies(\"is\"+data['pickup_datetime'].dt.weekday.apply(lambda x: weekday_names.get(x))))\n",
    "    #data = data.join(pd.get_dummies(\"is\"+data['pickup_datetime'].dt.month))\n",
    "    data['pickup_day_of_week']=data['pickup_datetime'].apply(lambda x:calendar.day_name[x.weekday()])\n",
    "    data['pickup_day_of_week'] = data['pickup_day_of_week'].apply(lambda x: encodeDays(x))\n",
    "    #^one-hot encoding of weekday\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/pandas/core/indexing.py:362: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[key] = _infer_fill_value(value)\n",
      "/usr/local/lib/python3.7/site-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of Training Data after cleaning  (978799, 13)\n",
      "Shape of Testing Data after cleaning (9914, 12)\n"
     ]
    }
   ],
   "source": [
    "train=clean_data(train)\n",
    "test=clean_data(test)\n",
    "print(\"Shape of Training Data after cleaning \",train.shape)\n",
    "print(\"Shape of Testing Data after cleaning\", test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.describe of                                   key  fare_amount     pickup_datetime  \\\n",
       "0         2009-06-15 17:26:21.0000001         4.50 2009-06-15 17:26:21   \n",
       "1         2010-01-05 16:52:16.0000002        16.90 2010-01-05 16:52:16   \n",
       "2        2011-08-18 00:35:00.00000049         5.70 2011-08-18 00:35:00   \n",
       "3         2012-04-21 04:30:42.0000001         7.70 2012-04-21 04:30:42   \n",
       "4       2010-03-09 07:51:00.000000135         5.30 2010-03-09 07:51:00   \n",
       "5         2011-01-06 09:50:45.0000002        12.10 2011-01-06 09:50:45   \n",
       "6         2012-11-20 20:35:00.0000001         7.50 2012-11-20 20:35:00   \n",
       "7        2012-01-04 17:22:00.00000081        16.50 2012-01-04 17:22:00   \n",
       "8       2012-12-03 13:10:00.000000125         9.00 2012-12-03 13:10:00   \n",
       "9        2009-09-02 01:11:00.00000083         8.90 2009-09-02 01:11:00   \n",
       "10        2012-04-08 07:30:50.0000002         5.30 2012-04-08 07:30:50   \n",
       "12        2009-11-06 01:04:03.0000002         4.10 2009-11-06 01:04:03   \n",
       "13      2013-07-02 19:54:00.000000232         7.00 2013-07-02 19:54:00   \n",
       "14        2011-04-05 17:11:05.0000001         7.70 2011-04-05 17:11:05   \n",
       "16       2014-02-19 07:22:00.00000074        12.50 2014-02-19 07:22:00   \n",
       "17      2009-07-22 16:08:00.000000163         5.30 2009-07-22 16:08:00   \n",
       "18       2010-07-07 14:52:00.00000044         5.30 2010-07-07 14:52:00   \n",
       "19        2014-12-06 20:36:22.0000008         4.00 2014-12-06 20:36:22   \n",
       "20      2010-09-07 13:18:00.000000175        10.50 2010-09-07 13:18:00   \n",
       "21        2013-02-12 12:15:46.0000002        11.50 2013-02-12 12:15:46   \n",
       "22        2009-08-06 18:17:23.0000001         4.50 2009-08-06 18:17:23   \n",
       "23      2010-12-06 12:29:00.000000199         4.90 2010-12-06 12:29:00   \n",
       "24       2009-12-10 15:37:00.00000020         6.10 2009-12-10 15:37:00   \n",
       "25       2011-06-21 16:15:00.00000073         7.30 2011-06-21 16:15:00   \n",
       "27      2011-06-28 19:47:00.000000168         4.50 2011-06-28 19:47:00   \n",
       "28        2012-05-04 06:11:20.0000002         9.30 2012-05-04 06:11:20   \n",
       "29       2013-08-11 00:52:00.00000026         4.50 2013-08-11 00:52:00   \n",
       "30      2014-02-19 16:03:00.000000160         5.50 2014-02-19 16:03:00   \n",
       "31        2015-06-21 21:46:34.0000001        22.54 2015-06-21 21:46:34   \n",
       "32      2009-01-09 16:10:00.000000170        31.90 2009-01-09 16:10:00   \n",
       "...                               ...          ...                 ...   \n",
       "999970    2013-10-25 23:36:36.0000005         8.50 2013-10-25 23:36:36   \n",
       "999971  2009-06-15 09:42:00.000000149        29.30 2009-06-15 09:42:00   \n",
       "999972    2009-08-10 17:47:43.0000005        10.10 2009-08-10 17:47:43   \n",
       "999973   2011-05-07 12:53:00.00000074        11.30 2011-05-07 12:53:00   \n",
       "999974  2013-10-16 07:54:00.000000161        21.50 2013-10-16 07:54:00   \n",
       "999975    2014-08-24 09:12:23.0000002        31.83 2014-08-24 09:12:23   \n",
       "999976    2011-01-06 14:50:45.0000003        12.50 2011-01-06 14:50:45   \n",
       "999977    2011-02-10 12:08:13.0000001        13.30 2011-02-10 12:08:13   \n",
       "999978    2012-02-22 00:47:14.0000001         6.50 2012-02-22 00:47:14   \n",
       "999979  2013-05-06 15:56:00.000000128        14.00 2013-05-06 15:56:00   \n",
       "999980    2014-04-07 07:00:13.0000001        18.50 2014-04-07 07:00:13   \n",
       "999981    2015-05-08 12:14:23.0000004        18.50 2015-05-08 12:14:23   \n",
       "999982   2011-11-10 06:02:00.00000018         4.50 2011-11-10 06:02:00   \n",
       "999983    2014-08-29 07:18:09.0000003         4.50 2014-08-29 07:18:09   \n",
       "999984    2014-11-02 14:09:53.0000001        44.33 2014-11-02 14:09:53   \n",
       "999985  2014-12-18 20:13:00.000000125         8.50 2014-12-18 20:13:00   \n",
       "999986    2015-06-02 09:21:33.0000001         7.00 2015-06-02 09:21:33   \n",
       "999987    2010-01-26 15:03:00.0000008         4.10 2010-01-26 15:03:00   \n",
       "999988   2011-05-14 07:21:00.00000014        25.30 2011-05-14 07:21:00   \n",
       "999989  2012-11-11 10:36:00.000000160         5.00 2012-11-11 10:36:00   \n",
       "999990   2009-09-22 16:43:00.00000052         6.10 2009-09-22 16:43:00   \n",
       "999991    2010-04-20 08:17:51.0000004         8.90 2010-04-20 08:17:51   \n",
       "999992    2011-01-21 10:57:27.0000001         5.70 2011-01-21 10:57:27   \n",
       "999993  2012-12-14 11:08:00.000000132        15.00 2012-12-14 11:08:00   \n",
       "999994    2014-06-29 22:57:55.0000001        20.00 2014-06-29 22:57:55   \n",
       "999995    2014-09-13 21:44:38.0000001         7.00 2014-09-13 21:44:38   \n",
       "999996    2010-09-20 14:50:37.0000002         7.30 2010-09-20 14:50:37   \n",
       "999997  2013-04-26 14:03:00.000000118        10.50 2013-04-26 14:03:00   \n",
       "999998   2011-07-08 00:29:00.00000099         6.90 2011-07-08 00:29:00   \n",
       "999999   2009-12-31 14:30:00.00000021         4.10 2009-12-31 14:30:00   \n",
       "\n",
       "        pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "0             -73.844311        40.721319         -73.841610   \n",
       "1             -74.016048        40.711303         -73.979268   \n",
       "2             -73.982738        40.761270         -73.991242   \n",
       "3             -73.987130        40.733143         -73.991567   \n",
       "4             -73.968095        40.768008         -73.956655   \n",
       "5             -74.000964        40.731630         -73.972892   \n",
       "6             -73.980002        40.751662         -73.973802   \n",
       "7             -73.951300        40.774138         -73.990095   \n",
       "8             -74.006462        40.726713         -73.993078   \n",
       "9             -73.980658        40.733873         -73.991540   \n",
       "10            -73.996335        40.737142         -73.980721   \n",
       "12            -73.991601        40.744712         -73.983081   \n",
       "13            -74.005360        40.728867         -74.008913   \n",
       "14            -74.001821        40.737547         -73.998060   \n",
       "16            -73.986430        40.760465         -73.988990   \n",
       "17            -73.981060        40.737690         -73.994177   \n",
       "18            -73.969505        40.784843         -73.958732   \n",
       "19            -73.979815        40.751902         -73.979446   \n",
       "20            -73.985382        40.747858         -73.978377   \n",
       "21            -73.957954        40.779252         -73.961250   \n",
       "22            -73.991707        40.770505         -73.985459   \n",
       "23            -74.000632        40.747473         -73.986672   \n",
       "24            -73.969622        40.756973         -73.981152   \n",
       "25            -73.991875        40.754437         -73.977230   \n",
       "27            -73.988893        40.760160         -73.986445   \n",
       "28            -73.989258        40.690835         -74.004133   \n",
       "29            -73.981020        40.737760         -73.980668   \n",
       "30            -73.976075        40.752422         -73.981082   \n",
       "31            -74.010483        40.717667         -73.985771   \n",
       "32            -73.873027        40.773883         -73.984545   \n",
       "...                  ...              ...                ...   \n",
       "999970        -73.967576        40.760106         -73.988016   \n",
       "999971        -73.776735        40.645323         -73.970758   \n",
       "999972        -73.983638        40.746203         -73.987377   \n",
       "999973        -73.952537        40.808417         -73.953065   \n",
       "999974        -73.991245        40.765387         -74.004995   \n",
       "999975        -73.873020        40.774041         -73.982572   \n",
       "999976        -73.981381        40.778593         -74.004399   \n",
       "999977        -74.015867        40.715233         -73.989395   \n",
       "999978        -73.984087        40.746916         -73.983599   \n",
       "999979        -74.002227        40.724750         -73.979082   \n",
       "999980        -74.010280        40.703269         -73.983382   \n",
       "999981        -73.956215        40.778343         -73.978371   \n",
       "999982        -73.985948        40.768127         -73.974000   \n",
       "999983        -73.991835        40.750038         -73.984214   \n",
       "999984        -73.872276        40.774264         -73.969262   \n",
       "999985        -74.003187        40.749022         -73.987785   \n",
       "999986        -73.975723        40.758533         -73.981537   \n",
       "999987        -73.982407        40.765418         -73.981803   \n",
       "999988        -73.937393        40.757975         -73.937393   \n",
       "999989        -73.982482        40.747582         -73.986402   \n",
       "999990        -73.963472        40.774893         -73.980038   \n",
       "999991        -73.973683        40.792300         -73.979603   \n",
       "999992        -73.976733        40.790480         -73.988215   \n",
       "999993        -73.975895        40.728558         -74.002202   \n",
       "999994        -73.872919        40.774106         -73.796524   \n",
       "999995        -73.976676        40.785630         -73.959196   \n",
       "999996        -73.992103        40.671385         -73.992103   \n",
       "999997        -73.978423        40.751135         -73.993397   \n",
       "999998        -73.980317        40.759482         -73.976832   \n",
       "999999        -74.006635        40.741598         -73.999450   \n",
       "\n",
       "        dropoff_latitude  passenger_count  pickup_day  pickup_hour  \\\n",
       "0              40.712278                1          15           17   \n",
       "1              40.782004                1           5           16   \n",
       "2              40.750562                2          18            0   \n",
       "3              40.758092                1          21            4   \n",
       "4              40.783762                1           9            7   \n",
       "5              40.758233                1           6            9   \n",
       "6              40.764842                1          20           20   \n",
       "7              40.751048                1           4           17   \n",
       "8              40.731628                1           3           13   \n",
       "9              40.758138                2           2            1   \n",
       "10             40.733559                1           8            7   \n",
       "12             40.744682                2           6            1   \n",
       "13             40.710907                1           2           19   \n",
       "14             40.722788                2           5           17   \n",
       "16             40.737075                1          19            7   \n",
       "17             40.728412                1          22           16   \n",
       "18             40.783357                1           7           14   \n",
       "19             40.755481                1           6           20   \n",
       "20             40.762070                1           7           13   \n",
       "21             40.758787                1          12           12   \n",
       "22             40.763671                1           6           18   \n",
       "23             40.740577                1           6           12   \n",
       "24             40.759712                1          10           15   \n",
       "25             40.774323                3          21           16   \n",
       "27             40.757857                3          28           19   \n",
       "28             40.725690                1           4            6   \n",
       "29             40.730497                2          11            0   \n",
       "30             40.759285                1          19           16   \n",
       "31             40.660366                1          21           21   \n",
       "32             40.769545                3           9           16   \n",
       "...                  ...              ...         ...          ...   \n",
       "999970         40.764855                1          25           23   \n",
       "999971         40.682848                5          15            9   \n",
       "999972         40.749008                2          10           17   \n",
       "999973         40.803848                5           7           12   \n",
       "999974         40.705362                2          16            7   \n",
       "999975         40.759972                2          24            9   \n",
       "999976         40.746174                1           6           14   \n",
       "999977         40.758612                0          10           12   \n",
       "999978         40.730348                1          22            0   \n",
       "999979         40.752920                1           6           15   \n",
       "999980         40.668774                1           7            7   \n",
       "999981         40.750416                1           8           12   \n",
       "999982         40.762892                1          10            6   \n",
       "999983         40.754428                1          29            7   \n",
       "999984         40.756591                2           2           14   \n",
       "999985         40.745847                1          18           20   \n",
       "999986         40.752602                1           2            9   \n",
       "999987         40.763305                1          26           15   \n",
       "999988         40.757975                5          14            7   \n",
       "999989         40.738122                5          11           10   \n",
       "999990         40.777478                1          22           16   \n",
       "999991         40.765361                1          20            8   \n",
       "999992         40.779298                1          21           10   \n",
       "999993         40.755675                2          14           11   \n",
       "999994         40.775041                2          29           22   \n",
       "999995         40.775114                1          13           21   \n",
       "999996         40.671385                2          20           14   \n",
       "999997         40.762197                5          26           14   \n",
       "999998         40.743122                1           8            0   \n",
       "999999         40.745702                1          31           14   \n",
       "\n",
       "        pickup_month  pickup_year  pickup_day_of_week  \n",
       "0                  6         2009                   1  \n",
       "1                  1         2010                   2  \n",
       "2                  8         2011                   4  \n",
       "3                  4         2012                   6  \n",
       "4                  3         2010                   2  \n",
       "5                  1         2011                   4  \n",
       "6                 11         2012                   2  \n",
       "7                  1         2012                   3  \n",
       "8                 12         2012                   1  \n",
       "9                  9         2009                   3  \n",
       "10                 4         2012                   0  \n",
       "12                11         2009                   5  \n",
       "13                 7         2013                   2  \n",
       "14                 4         2011                   2  \n",
       "16                 2         2014                   3  \n",
       "17                 7         2009                   3  \n",
       "18                 7         2010                   3  \n",
       "19                12         2014                   6  \n",
       "20                 9         2010                   2  \n",
       "21                 2         2013                   2  \n",
       "22                 8         2009                   4  \n",
       "23                12         2010                   1  \n",
       "24                12         2009                   4  \n",
       "25                 6         2011                   2  \n",
       "27                 6         2011                   2  \n",
       "28                 5         2012                   5  \n",
       "29                 8         2013                   0  \n",
       "30                 2         2014                   3  \n",
       "31                 6         2015                   0  \n",
       "32                 1         2009                   5  \n",
       "...              ...          ...                 ...  \n",
       "999970            10         2013                   5  \n",
       "999971             6         2009                   1  \n",
       "999972             8         2009                   1  \n",
       "999973             5         2011                   6  \n",
       "999974            10         2013                   3  \n",
       "999975             8         2014                   0  \n",
       "999976             1         2011                   4  \n",
       "999977             2         2011                   4  \n",
       "999978             2         2012                   3  \n",
       "999979             5         2013                   1  \n",
       "999980             4         2014                   1  \n",
       "999981             5         2015                   5  \n",
       "999982            11         2011                   4  \n",
       "999983             8         2014                   5  \n",
       "999984            11         2014                   0  \n",
       "999985            12         2014                   4  \n",
       "999986             6         2015                   2  \n",
       "999987             1         2010                   2  \n",
       "999988             5         2011                   6  \n",
       "999989            11         2012                   0  \n",
       "999990             9         2009                   2  \n",
       "999991             4         2010                   2  \n",
       "999992             1         2011                   5  \n",
       "999993            12         2012                   5  \n",
       "999994             6         2014                   0  \n",
       "999995             9         2014                   6  \n",
       "999996             9         2010                   1  \n",
       "999997             4         2013                   5  \n",
       "999998             7         2011                   5  \n",
       "999999            12         2009                   4  \n",
       "\n",
       "[978799 rows x 13 columns]>"
      ]
     },
     "execution_count": 335,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [],
   "source": [
    "def processDataForModelling(data,target,drop_cols,is_train=True,split=0.25):\n",
    "    data_1=data.drop(drop_cols,axis=1)\n",
    "    if is_train==True:\n",
    "        X=data_1.drop([target],axis=1)\n",
    "        y=data_1[target]\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=split,random_state=123)\n",
    "        \n",
    "        print(\"Shape of Training Features\",X_train.shape)\n",
    "        print(\"Shape of Validation Features \",X_test.shape)\n",
    "        \n",
    "        return X_train, X_test, y_train, y_test\n",
    "    else:\n",
    "        print (\"Shape of Test Data\",data_1.shape)\n",
    "        return data_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of Training Features (783039, 10)\n",
      "Shape of Validation Features  (195760, 10)\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test=processDataForModelling(train,'fare_amount',drop_cols=['key','pickup_datetime'],is_train=True,split=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.describe of         pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "975541        -73.870807        40.773718         -73.987162   \n",
       "874090        -73.998912        40.719423         -73.994190   \n",
       "543271        -73.994255        40.730018         -73.994583   \n",
       "412003        -74.002666        40.726937         -73.982204   \n",
       "401094        -73.983312        40.754823         -73.946593   \n",
       "351836        -73.985810        40.740675         -73.983197   \n",
       "812933        -73.963642        40.774420         -73.963893   \n",
       "293926        -73.974355        40.759955         -73.955708   \n",
       "4232          -73.989004        40.730487         -73.991177   \n",
       "599955        -73.981979        40.773564         -73.981548   \n",
       "287151        -74.005470        40.739407         -74.001587   \n",
       "668764        -73.990278        40.752050         -73.996807   \n",
       "211805        -74.002368        40.733675         -73.999120   \n",
       "2980          -73.966778        40.753462         -73.976775   \n",
       "228737        -73.980430        40.765577         -73.998343   \n",
       "323339        -73.873161        40.774178         -73.952751   \n",
       "734499        -74.010593        40.718125         -74.001407   \n",
       "550978        -73.994052        40.732056         -73.997930   \n",
       "190116        -73.988527        40.737188         -73.972878   \n",
       "448623        -73.996605        40.737488         -73.994328   \n",
       "136971        -73.973059        40.763117         -73.977684   \n",
       "850779        -73.954991        40.768309         -73.967060   \n",
       "817641        -74.216747        40.581768         -74.216747   \n",
       "164868        -73.980093        40.739277         -73.984790   \n",
       "164450        -73.992617        40.728367         -73.976122   \n",
       "366363        -74.000167        40.736983         -73.993618   \n",
       "668145        -73.985558        40.743894         -73.986022   \n",
       "661770        -73.979035        40.762902         -73.967127   \n",
       "333549        -73.994735        40.725884         -73.953419   \n",
       "572062        -73.939921        40.799580         -73.953743   \n",
       "...                  ...              ...                ...   \n",
       "878513        -73.964965        40.772557         -73.963050   \n",
       "888216        -73.989020        40.761187         -73.970543   \n",
       "668709        -73.988333        40.825148         -73.988333   \n",
       "417037        -73.994508        40.750732         -73.948055   \n",
       "768246        -73.985341        40.752862         -73.965671   \n",
       "472121        -73.966707        40.753105         -74.007085   \n",
       "572100        -73.982603        40.745522         -73.982492   \n",
       "419922        -73.981265        40.774097         -73.979952   \n",
       "773452        -73.929771        40.812257         -73.929784   \n",
       "200984        -73.952813        40.783240         -73.967132   \n",
       "525448        -73.999919        40.733047         -73.975636   \n",
       "703949        -73.972162        40.762768         -73.964065   \n",
       "608438        -73.985079        40.742041         -74.006196   \n",
       "121481        -73.997630        40.724277         -74.006970   \n",
       "255309        -73.983890        40.759742         -74.001842   \n",
       "961038        -73.986527        40.758086         -73.992176   \n",
       "458398        -73.874856        40.774021         -73.977661   \n",
       "67078         -73.957728        40.761688         -73.972468   \n",
       "156657        -73.975630        40.754940         -73.955130   \n",
       "225173        -73.998143        40.761154         -74.009715   \n",
       "47217         -74.002167        40.729982         -73.993450   \n",
       "667557        -73.971662        40.748942         -73.984043   \n",
       "685238        -74.005066        40.719482         -74.006203   \n",
       "198488        -73.954053        40.778915         -73.974892   \n",
       "925923        -73.971100        40.783040         -73.966662   \n",
       "196637        -73.986865        40.760197         -74.002370   \n",
       "18126         -73.985069        40.728078         -73.981547   \n",
       "28643         -73.978818        40.777291         -73.969107   \n",
       "283859        -73.970187        40.762440         -73.988370   \n",
       "790278        -73.952582        40.706606         -73.964954   \n",
       "\n",
       "        dropoff_latitude  passenger_count  pickup_day  pickup_hour  \\\n",
       "975541         40.745317                1           8           15   \n",
       "874090         40.713633                1          21           19   \n",
       "543271         40.740287                1          19           12   \n",
       "412003         40.752569                1           3           15   \n",
       "401094         40.806285                1          27           14   \n",
       "351836         40.756600                1          16           14   \n",
       "812933         40.764422                1           3            9   \n",
       "293926         40.767601                1           2           22   \n",
       "4232           40.719543                1          12           22   \n",
       "599955         40.761777                1          30           16   \n",
       "287151         40.729313                1          23            1   \n",
       "668764         40.742587                1           6            0   \n",
       "211805         40.740052                5           4           23   \n",
       "2980           40.745490                2           3           18   \n",
       "228737         40.740277                1          14           16   \n",
       "323339         40.731827                2           5           16   \n",
       "734499         40.715708                1          24            8   \n",
       "550978         40.740357                1          20           22   \n",
       "190116         40.753482                1          23            9   \n",
       "448623         40.761103                2          30           19   \n",
       "136971         40.776826                1           6           12   \n",
       "850779         40.759273                1          18           19   \n",
       "817641         40.581768                2           9           19   \n",
       "164868         40.741247                1          26            9   \n",
       "164450         40.760224                1          19            1   \n",
       "366363         40.734204                1          29            9   \n",
       "668145         40.740163                2          11            4   \n",
       "661770         40.771605                3          22           12   \n",
       "333549         40.779399                1          18            2   \n",
       "572062         40.800398                1          21           22   \n",
       "...                  ...              ...         ...          ...   \n",
       "878513         40.771857                1          25           16   \n",
       "888216         40.759347                5          25           19   \n",
       "668709         40.825148                1          25            0   \n",
       "417037         40.800382                6          20           15   \n",
       "768246         40.758482                2           6           23   \n",
       "472121         40.741737                6          22            1   \n",
       "572100         40.756057                5          23           12   \n",
       "419922         40.781043                3          30           23   \n",
       "773452         40.812266                1          19           15   \n",
       "200984         40.765915                5          13           16   \n",
       "525448         40.735614                1          13           22   \n",
       "703949         40.773487                2          14           11   \n",
       "608438         40.743831                1          14            1   \n",
       "121481         40.739397                5          26           14   \n",
       "255309         40.717118                1           2           23   \n",
       "961038         40.748095                2          10           10   \n",
       "458398         40.751931                1           1            9   \n",
       "67078          40.749864                1          16           10   \n",
       "156657         40.769010                1          18           23   \n",
       "225173         40.705537                1          29           19   \n",
       "47217          40.678290                2          25            2   \n",
       "667557         40.754383                1          12            1   \n",
       "685238         40.734257                1          23            3   \n",
       "198488         40.741957                1          28            5   \n",
       "925923         40.764217                1          23           14   \n",
       "196637         40.760842                1          10           18   \n",
       "18126          40.766891                1          25           18   \n",
       "28643          40.755846                1          10           17   \n",
       "283859         40.731689                1          23           20   \n",
       "790278         40.710863                4          25           23   \n",
       "\n",
       "        pickup_month  pickup_year  pickup_day_of_week  \n",
       "975541             6         2010                   2  \n",
       "874090             3         2009                   6  \n",
       "543271             3         2012                   1  \n",
       "412003             8         2012                   5  \n",
       "401094             9         2010                   1  \n",
       "351836            10         2013                   3  \n",
       "812933             4         2014                   4  \n",
       "293926             3         2010                   2  \n",
       "4232              10         2009                   1  \n",
       "599955            11         2010                   2  \n",
       "287151             5         2015                   6  \n",
       "668764            10         2012                   6  \n",
       "211805             4         2010                   0  \n",
       "2980               2         2009                   2  \n",
       "228737             6         2014                   6  \n",
       "323339             4         2015                   0  \n",
       "734499             1         2013                   4  \n",
       "550978             2         2014                   4  \n",
       "190116             5         2011                   1  \n",
       "448623             9         2009                   3  \n",
       "136971            12         2009                   0  \n",
       "850779             3         2012                   0  \n",
       "817641            11         2011                   3  \n",
       "164868             5         2009                   2  \n",
       "164450             5         2009                   2  \n",
       "366363             1         2009                   4  \n",
       "668145             7         2010                   0  \n",
       "661770            10         2013                   2  \n",
       "333549             9         2009                   5  \n",
       "572062             4         2011                   4  \n",
       "...              ...          ...                 ...  \n",
       "878513             2         2014                   2  \n",
       "888216             8         2013                   0  \n",
       "668709             7         2012                   3  \n",
       "417037             6         2014                   5  \n",
       "768246             3         2014                   4  \n",
       "472121             1         2012                   0  \n",
       "572100            12         2010                   4  \n",
       "419922            10         2009                   5  \n",
       "773452             7         2012                   4  \n",
       "200984            12         2010                   1  \n",
       "525448             8         2013                   2  \n",
       "703949             9         2011                   3  \n",
       "608438             7         2012                   6  \n",
       "121481            10         2014                   0  \n",
       "255309             8         2010                   1  \n",
       "961038             7         2012                   2  \n",
       "458398             8         2012                   3  \n",
       "67078             11         2012                   5  \n",
       "156657            12         2014                   4  \n",
       "225173             6         2014                   0  \n",
       "47217              5         2012                   5  \n",
       "667557             8         2010                   4  \n",
       "685238             4         2015                   4  \n",
       "198488             8         2014                   4  \n",
       "925923             1         2012                   1  \n",
       "196637             6         2014                   2  \n",
       "18126              9         2011                   0  \n",
       "28643              1         2010                   0  \n",
       "283859            12         2010                   4  \n",
       "790278             5         2013                   6  \n",
       "\n",
       "[783039 rows x 10 columns]>"
      ]
     },
     "execution_count": 338,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.describe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of Test Data (9914, 10)\n"
     ]
    }
   ],
   "source": [
    "test_data=processDataForModelling(test,'fare_amount',drop_cols=['key','pickup_datetime'],is_train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Basline RMSE of Validation data : 9.659505711183623\n"
     ]
    }
   ],
   "source": [
    "avg_fare=round(np.mean(y_train),2)\n",
    "baseline_pred=np.repeat(avg_fare,y_test.shape[0])\n",
    "baseline_rmse=np.sqrt(mean_squared_error(baseline_pred, y_test))\n",
    "print(\"Basline RMSE of Validation data :\",baseline_rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE for Linear Regression is  8.219077313138138\n"
     ]
    }
   ],
   "source": [
    "lm = LinearRegression()\n",
    "lm.fit(X_train,y_train)\n",
    "y_pred=np.round(lm.predict(X_test),2)\n",
    "lm_rmse=np.sqrt(mean_squared_error(y_pred, y_test))\n",
    "print(\"RMSE for Linear Regression is \",lm_rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 4 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building tree 1 of 100building tree 2 of 100building tree 3 of 100\n",
      "building tree 4 of 100\n",
      "\n",
      "\n",
      "building tree 5 of 100\n",
      "building tree 6 of 100\n",
      "building tree 7 of 100\n",
      "building tree 8 of 100\n",
      "building tree 9 of 100\n",
      "building tree 10 of 100\n",
      "building tree 11 of 100\n",
      "building tree 12 of 100\n",
      "building tree 13 of 100\n",
      "building tree 14 of 100\n",
      "building tree 15 of 100\n",
      "building tree 16 of 100\n",
      "building tree 17 of 100\n",
      "building tree 18 of 100\n",
      "building tree 19 of 100\n",
      "building tree 20 of 100\n",
      "building tree 21 of 100\n",
      "building tree 22 of 100building tree 23 of 100\n",
      "\n",
      "building tree 24 of 100\n",
      "building tree 25 of 100\n",
      "building tree 26 of 100\n",
      "building tree 27 of 100\n",
      "building tree 28 of 100\n",
      "building tree 29 of 100\n",
      "building tree 30 of 100\n",
      "building tree 31 of 100\n",
      "building tree 32 of 100\n",
      "building tree 33 of 100\n",
      "building tree 34 of 100\n",
      "building tree 35 of 100\n",
      "building tree 36 of 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:  2.4min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building tree 37 of 100\n",
      "building tree 38 of 100\n",
      "building tree 39 of 100\n",
      "building tree 40 of 100\n",
      "building tree 41 of 100\n",
      "building tree 42 of 100\n",
      "building tree 43 of 100\n",
      "building tree 44 of 100\n",
      "building tree 45 of 100\n",
      "building tree 46 of 100\n",
      "building tree 47 of 100\n",
      "building tree 48 of 100\n",
      "building tree 49 of 100\n",
      "building tree 50 of 100\n",
      "building tree 51 of 100\n",
      "building tree 52 of 100\n",
      "building tree 53 of 100\n",
      "building tree 54 of 100\n",
      "building tree 55 of 100\n",
      "building tree 56 of 100\n",
      "building tree 57 of 100\n",
      "building tree 58 of 100\n",
      "building tree 59 of 100\n",
      "building tree 60 of 100\n",
      "building tree 61 of 100\n",
      "building tree 62 of 100\n",
      "building tree 63 of 100\n",
      "building tree 64 of 100\n",
      "building tree 65 of 100\n",
      "building tree 66 of 100\n",
      "building tree 67 of 100\n",
      "building tree 68 of 100\n",
      "building tree 69 of 100\n",
      "building tree 70 of 100\n",
      "building tree 71 of 100\n",
      "building tree 72 of 100\n",
      "building tree 73 of 100\n",
      "building tree 74 of 100\n",
      "building tree 75 of 100\n",
      "building tree 76 of 100\n",
      "building tree 77 of 100\n",
      "building tree 78 of 100\n",
      "building tree 79 of 100\n",
      "building tree 80 of 100\n",
      "building tree 81 of 100\n",
      "building tree 82 of 100\n",
      "building tree 83 of 100\n",
      "building tree 84 of 100\n",
      "building tree 85 of 100\n",
      "building tree 86 of 100\n",
      "building tree 87 of 100\n",
      "building tree 88 of 100\n",
      "building tree 89 of 100\n",
      "building tree 90 of 100\n",
      "building tree 91 of 100\n",
      "building tree 92 of 100\n",
      "building tree 93 of 100\n",
      "building tree 94 of 100\n",
      "building tree 95 of 100\n",
      "building tree 96 of 100\n",
      "building tree 97 of 100\n",
      "building tree 98 of 100\n",
      "building tree 99 of 100\n",
      "building tree 100 of 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:  6.9min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  33 tasks      | elapsed:    4.0s\n",
      "[Parallel(n_jobs=4)]: Done 100 out of 100 | elapsed:   11.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE for Random Forest is  3.9809285947728537\n",
      "[0.29623826 0.12151069 0.37385929 0.133674   0.00401601 0.01286484\n",
      " 0.01483161 0.01041534 0.02494751 0.00764245]\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestRegressor(n_estimators = 100, random_state = 1,n_jobs=-1, verbose=2)\n",
    "rf.fit(X_train,y_train)\n",
    "rf_pred= rf.predict(X_test)\n",
    "rf_rmse=np.sqrt(mean_squared_error(rf_pred, y_test))\n",
    "print(\"RMSE for Random Forest is \",rf_rmse)\n",
    "print(rf.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}