Predicting_House_Prices.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.options.display.max_columns = 999\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn import linear_model\n",
    "from sklearn.model_selection import cross_val_score, KFold\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "\n",
    "\n",
    "df = pd.read_csv('AmesHousing.tsv',sep='\\t')\n",
    "df = df.iloc[np.random.permutation(len(df))]\n",
    "df = df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/dataquest/system/env/python3/lib/python3.4/site-packages/sklearn/metrics/scorer.py:90: DeprecationWarning:\n",
      "\n",
      "Scoring method mean_squared_error was renamed to neg_mean_squared_error in version 0.18 and will be removed in 0.20.\n",
      "\n",
      "/dataquest/system/env/python3/lib/python3.4/site-packages/sklearn/metrics/scorer.py:90: DeprecationWarning:\n",
      "\n",
      "Scoring method mean_squared_error was renamed to neg_mean_squared_error in version 0.18 and will be removed in 0.20.\n",
      "\n",
      "/dataquest/system/env/python3/lib/python3.4/site-packages/sklearn/metrics/scorer.py:90: DeprecationWarning:\n",
      "\n",
      "Scoring method mean_squared_error was renamed to neg_mean_squared_error in version 0.18 and will be removed in 0.20.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-1.36820245e+09 -1.48274187e+09 -1.37677072e+09 -1.51688714e+09]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/dataquest/system/env/python3/lib/python3.4/site-packages/sklearn/metrics/scorer.py:90: DeprecationWarning:\n",
      "\n",
      "Scoring method mean_squared_error was renamed to neg_mean_squared_error in version 0.18 and will be removed in 0.20.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def transform_features(df):\n",
    "    num_missing = df.isnull().sum()\n",
    "    drop_missing_cols = num_missing[(num_missing > len(df)/20)].sort_values()\n",
    "    df = df.drop(drop_missing_cols.index, axis=1)\n",
    "    # Dropping columns with percentage of null\n",
    "    \n",
    "    text_mv_counts = df.select_dtypes(include=['object']).isnull().sum().sort_values(ascending=False)\n",
    "    drop_missing_cols_2 = text_mv_counts[text_mv_counts > 0]\n",
    "    df = df.drop(drop_missing_cols_2.index, axis=1)\n",
    "    # Dropping columns with str dtype\n",
    "    num_missing = df.select_dtypes(include=['int', 'float']).isnull().sum()\n",
    "    fixable_numeric_cols = num_missing[(num_missing < len(df)/20) & (num_missing > 0)].sort_values()\n",
    "    replacement_values_dict = df[fixable_numeric_cols.index].mode().to_dict(orient='records')[0]\n",
    "    df = df.fillna(replacement_values_dict)\n",
    "    # I need to understand this better\n",
    "    years_sold = df['Yr Sold'] - df['Year Built']\n",
    "    years_since_remod = df['Yr Sold'] - df['Year Remod/Add']\n",
    "    df['Years Before Sale'] = years_sold\n",
    "    df['Years Since Remod'] = years_since_remod\n",
    "    df = df.drop([1702, 2180, 2181], axis=0)\n",
    "    # Creating and Renaming new columns, dropping old columns\n",
    "\n",
    "    df = df.drop([\"PID\", \"Order\", \"Mo Sold\", \"Sale Condition\", \"Sale Type\", \"Year Built\", \"Year Remod/Add\"], axis=1)\n",
    "    #dropping data leakage or irrelevant column names\n",
    "    return df\n",
    "\n",
    "def select_features(df, coeff_threshold=0.4, uniq_threshold=10):\n",
    "    numerical_df = df.select_dtypes(include=['int', 'float'])\n",
    "    abs_corr_coeffs = numerical_df.corr()['SalePrice'].abs().sort_values()\n",
    "    df = df.drop(abs_corr_coeffs[abs_corr_coeffs < coeff_threshold].index, axis=1)\n",
    "    \n",
    "    nominal_features = [\"PID\", \"MS SubClass\", \"MS Zoning\", \"Street\", \"Alley\", \"Land Contour\", \"Lot Config\", \"Neighborhood\", \n",
    "                    \"Condition 1\", \"Condition 2\", \"Bldg Type\", \"House Style\", \"Roof Style\", \"Roof Matl\", \"Exterior 1st\", \n",
    "                    \"Exterior 2nd\", \"Mas Vnr Type\", \"Foundation\", \"Heating\", \"Central Air\", \"Garage Type\", \n",
    "                    \"Misc Feature\", \"Sale Type\", \"Sale Condition\"]\n",
    "    \n",
    "    transform_cat_cols = []\n",
    "    for col in nominal_features:\n",
    "        if col in df.columns:\n",
    "            transform_cat_cols.append(col)\n",
    "\n",
    "    uniqueness_counts = df[transform_cat_cols].apply(lambda col: len(col.value_counts())).sort_values()\n",
    "    drop_nonuniq_cols = uniqueness_counts[uniqueness_counts > uniq_threshold].index\n",
    "    df = df.drop(drop_nonuniq_cols, axis=1)\n",
    "    \n",
    "    text_cols = df.select_dtypes(include=['object'])\n",
    "    for col in text_cols:\n",
    "        df[col] = df[col].astype('category')\n",
    "    df = pd.concat([df, pd.get_dummies(df.select_dtypes(include=['category']))], axis=1).drop(text_cols,axis=1)\n",
    "    \n",
    "    return df\n",
    "\n",
    "def train_and_test(df,k=0):\n",
    "    df = df.select_dtypes(include=['integer', 'float'])\n",
    "    features = df.columns.drop(\"SalePrice\")\n",
    "    if k == 0:\n",
    "        train = df[:1460]\n",
    "        test = df[1460:]\n",
    "\n",
    "        ## You can use `pd.DataFrame.select_dtypes()` to specify column types\n",
    "        ## and return only those columns as a data frame.\n",
    "        ## You can use `pd.Series.drop()` to drop a value.\n",
    "        lr = linear_model.LinearRegression()\n",
    "        lr.fit(train[features], train[\"SalePrice\"])\n",
    "        predictions = lr.predict(test[features])\n",
    "        mse = mean_squared_error(test[\"SalePrice\"], predictions)\n",
    "        rmse = np.sqrt(mse)\n",
    "\n",
    "        return rmse\n",
    "    elif k == 1:\n",
    "        fold_one = df[:1460]\n",
    "        fold_two = df[1460:]\n",
    "        ## You can use `pd.Series.drop()` to drop a value.\n",
    "        model = KNeighborsRegressor()\n",
    "        \n",
    "        test_one = model.fit(fold_one[features], fold_one[\"SalePrice\"])\n",
    "        fold_two_prediction = test_one.predict(fold_two[features])\n",
    "         \n",
    "        simple_cross_rmse_one = mean_squared_error(fold_two[\"SalePrice\"],fold_two_prediction)**(1/2)\n",
    "        test_two = model.fit(fold_two[features], fold_two[\"SalePrice\"])\n",
    "        \n",
    "        fold_one_prediction = test_two.predict(fold_one[features])\n",
    "        simple_cross_rmse_two = mean_squared_error(fold_one[\"SalePrice\"],fold_one_prediction)\n",
    "        \n",
    "        return simple_cross_rmse_one,simple_cross_rmse_one                      \n",
    "    else:\n",
    "        knn = KNeighborsRegressor()\n",
    "        kf = KFold(n_splits=k)\n",
    "        mses = cross_val_score(estimator=knn, X=df[features],y=df['SalePrice'], scoring='mean_squared_error',cv=kf)\n",
    "        return mses\n",
    "       \n",
    "    \n",
    "transform_df = transform_features(df)\n",
    "filtered_df = select_features(transform_df)\n",
    "rmse = train_and_test(filtered_df,k=4)\n",
    "rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}